[llvm] [AMDGPU] Port Atomic-Optimizer to use Wave Reduction Intrinsics (PR #211757)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 29 01:29:33 PDT 2026
https://github.com/easyonaadit updated https://github.com/llvm/llvm-project/pull/211757
>From ad91a2a955ceda43e591fadb2529d85c1022cea5 Mon Sep 17 00:00:00 2001
From: Aaditya <Aaditya.AlokDeshpande at amd.com>
Date: Fri, 24 Jul 2026 11:25:43 +0530
Subject: [PATCH 1/3] [AMDGPU] Port Atomic-Optimizer to use Wave Reduction
Intrinsics
Currently the atomic optimizer creates reductions via
intrinsics, and introduces new control flows.
Replace this sub-target dependent logic with existing
wave reduction intrinsics, which get lowered in the
backend.
This patch ports the uniform-value and divergent-no-return-value cases.
---
.../Target/AMDGPU/AMDGPUAtomicOptimizer.cpp | 95 +-
.../atomic_optimizations_mul_one.ll | 138 +-
.../AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll | 429 +-
.../global-atomic-fadd.f32-no-rtn.ll | 163 +-
.../AMDGPU/atomic-optimizer-promote-i8.ll | 18 +-
.../atomic_optimization_split_dt_update.ll | 28 +-
.../AMDGPU/atomic_optimizations_buffer.ll | 542 +-
.../atomic_optimizations_global_pointer.ll | 2461 ++--
.../atomic_optimizations_local_pointer.ll | 2949 ++---
.../atomic_optimizations_pixelshader.ll | 76 +-
.../AMDGPU/atomic_optimizations_raw_buffer.ll | 542 +-
.../atomic_optimizations_struct_buffer.ll | 586 +-
.../AMDGPU/atomics-hw-remarks-gfx90a.ll | 2 +-
.../AMDGPU/global-atomic-fadd.f32-no-rtn.ll | 84 +-
.../test/CodeGen/AMDGPU/global-atomic-scan.ll | 408 +-
.../global-atomicrmw-fadd-wrong-subtarget.ll | 39 +-
.../global-atomics-fp-wrong-subtarget.ll | 14 +-
.../AMDGPU/global_atomic_optimizer_fp_rtn.ll | 1300 ++-
.../AMDGPU/global_atomics_iterative_scan.ll | 36 +-
.../global_atomics_iterative_scan_fp.ll | 132 +-
.../global_atomics_optimizer_fp_no_rtn.ll | 1296 +--
.../AMDGPU/global_atomics_scan_fadd.ll | 9978 ++++++++---------
.../AMDGPU/global_atomics_scan_fmax.ll | 3796 +++----
.../AMDGPU/global_atomics_scan_fmin.ll | 3796 +++----
.../AMDGPU/global_atomics_scan_fsub.ll | 9502 ++++++++--------
.../insert_waitcnt_for_precise_memory.ll | 161 +-
.../CodeGen/AMDGPU/local-atomicrmw-fadd.ll | 891 +-
27 files changed, 19051 insertions(+), 20411 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
index b26db90c69096..fffd01efca39b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
@@ -642,6 +642,36 @@ static Constant *getIdentityValueForAtomicOp(Type *const Ty,
}
}
+static Intrinsic::ID getWaveReductionIntrinsic(AtomicRMWInst::BinOp Op) {
+ switch (Op) {
+ default:
+ llvm_unreachable(
+ "Atomic Op yet to be ported to use Wave Reduction intrinsics.");
+ case AtomicRMWInst::Add:
+ case AtomicRMWInst::Sub:
+ return Intrinsic::amdgcn_wave_reduce_add;
+ case AtomicRMWInst::FAdd:
+ case AtomicRMWInst::FSub:
+ return Intrinsic::amdgcn_wave_reduce_fadd;
+ case AtomicRMWInst::And:
+ return Intrinsic::amdgcn_wave_reduce_and;
+ case AtomicRMWInst::Or:
+ return Intrinsic::amdgcn_wave_reduce_or;
+ case AtomicRMWInst::Xor:
+ return Intrinsic::amdgcn_wave_reduce_xor;
+ case AtomicRMWInst::UMax:
+ case AtomicRMWInst::Max:
+ return Intrinsic::amdgcn_wave_reduce_max;
+ case AtomicRMWInst::FMax:
+ return Intrinsic::amdgcn_wave_reduce_fmax;
+ case AtomicRMWInst::UMin:
+ case AtomicRMWInst::Min:
+ return Intrinsic::amdgcn_wave_reduce_min;
+ case AtomicRMWInst::FMin:
+ return Intrinsic::amdgcn_wave_reduce_fmin;
+ }
+}
+
static Value *buildMul(IRBuilder<> &B, Value *LHS, Value *RHS) {
const ConstantInt *CI = dyn_cast<ConstantInt>(LHS);
return (CI && CI->isOne()) ? RHS : B.CreateMul(LHS, RHS);
@@ -732,6 +762,8 @@ void AMDGPUAtomicOptimizerImpl::optimizeAtomic(Instruction &I,
Function *F = I.getFunction();
LLVMContext &C = F->getContext();
+ const bool NeedResult = !I.use_empty();
+ const bool UseWaveReductionIntrinsic = !ValDivergent || !NeedResult;
// For atomic sub, perform scan with add operation and allow one lane to
// subtract the reduced value later.
@@ -746,16 +778,20 @@ void AMDGPUAtomicOptimizerImpl::optimizeAtomic(Instruction &I,
Value *ExclScan = nullptr;
Value *NewV = nullptr;
- const bool NeedResult = !I.use_empty();
-
BasicBlock *ComputeLoop = nullptr;
BasicBlock *ComputeEnd = nullptr;
- // If we have a divergent value in each lane, we need to combine the value
- // using DPP.
- if (ValDivergent) {
+ if (UseWaveReductionIntrinsic) {
+ // Build reductions with wave-reduce intrinsics.
+ unsigned Strategy = (ScanImpl == ScanOptions::DPP) ? 2 : 1;
+ Intrinsic::ID WaveRedIntrinsic = getWaveReductionIntrinsic(Op);
+ NewV = B.CreateIntrinsic(WaveRedIntrinsic, Ty, {V, B.getInt32(Strategy)});
+ } else {
+ // If we have a divergent value in each lane, we need to combine the value
+ // using DPP.
+ assert(ValDivergent && NeedResult);
if (ScanImpl == ScanOptions::DPP) {
- // First we need to set all inactive invocations to the identity value, so
- // that they can correctly contribute to the final result.
+ // First we need to set all inactive invocations to the identity value,
+ // so that they can correctly contribute to the final result.
NewV =
B.CreateIntrinsic(Intrinsic::amdgcn_set_inactive, Ty, {V, Identity});
if (!NeedResult && ST.hasPermlane16Insts()) {
@@ -785,49 +821,6 @@ void AMDGPUAtomicOptimizerImpl::optimizeAtomic(Instruction &I,
} else {
llvm_unreachable("Atomic Optimzer is disabled for None strategy");
}
- } else {
- switch (Op) {
- default:
- llvm_unreachable("Unhandled atomic op");
-
- case AtomicRMWInst::Add:
- case AtomicRMWInst::Sub: {
- // The new value we will be contributing to the atomic operation is the
- // old value times the number of active lanes.
- Value *const Ctpop = B.CreateIntCast(
- B.CreateUnaryIntrinsic(Intrinsic::ctpop, Ballot), Ty, false);
- NewV = buildMul(B, V, Ctpop);
- break;
- }
- case AtomicRMWInst::FAdd:
- case AtomicRMWInst::FSub: {
- Value *const Ctpop = B.CreateIntCast(
- B.CreateUnaryIntrinsic(Intrinsic::ctpop, Ballot), Int32Ty, false);
- Value *const CtpopFP = B.CreateUIToFP(Ctpop, Ty);
- NewV = B.CreateFMul(V, CtpopFP);
- break;
- }
- case AtomicRMWInst::And:
- case AtomicRMWInst::Or:
- case AtomicRMWInst::Max:
- case AtomicRMWInst::Min:
- case AtomicRMWInst::UMax:
- case AtomicRMWInst::UMin:
- case AtomicRMWInst::FMin:
- case AtomicRMWInst::FMax:
- // These operations with a uniform value are idempotent: doing the atomic
- // operation multiple times has the same effect as doing it once.
- NewV = V;
- break;
-
- case AtomicRMWInst::Xor:
- // The new value we will be contributing to the atomic operation is the
- // old value times the parity of the number of active lanes.
- Value *const Ctpop = B.CreateIntCast(
- B.CreateUnaryIntrinsic(Intrinsic::ctpop, Ballot), Ty, false);
- NewV = buildMul(B, V, B.CreateAnd(Ctpop, 1));
- break;
- }
}
// We only want a single lane to enter our new control flow, and we do this
@@ -855,7 +848,7 @@ void AMDGPUAtomicOptimizerImpl::optimizeAtomic(Instruction &I,
// ComputeEnd block. We also need to set up predecessor to next block when
// single lane done updating the final reduced value.
BasicBlock *Predecessor = nullptr;
- if (ValDivergent && ScanImpl == ScanOptions::Iterative) {
+ if (NeedResult && ValDivergent && ScanImpl == ScanOptions::Iterative) {
// Move terminator from I's block to ComputeEnd block.
//
// OriginalBB is known to have a branch as terminator because
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll
index 4d08b2c582d2b..760b0a1a1a2c8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: opt -S -mtriple=amdgpu-- -passes=amdgpu-atomic-optimizer %s | FileCheck -check-prefix=IR %s
; RUN: llc -global-isel -mtriple=amdgpu6.00-- < %s | FileCheck -check-prefix=GCN %s
@@ -23,8 +23,7 @@ define amdgpu_cs void @atomic_add(<4 x i32> inreg %arg) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
+; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 1, i32 1)
; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP8]], label %[[BB9:.*]], label %[[BB11:.*]]
; IR: [[BB9]]:
@@ -35,14 +34,15 @@ define amdgpu_cs void @atomic_add(<4 x i32> inreg %arg) {
;
; GCN-LABEL: atomic_add:
; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
; GCN-NEXT: s_mov_b64 s[4:5], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s4, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s5, v0
+; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GCN-NEXT: s_cbranch_execz .LBB0_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
+; GCN-NEXT: s_mul_i32 s4, s4, 1
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: buffer_atomic_add v0, v1, s[0:3], 0 idxen
@@ -63,8 +63,7 @@ define amdgpu_cs void @atomic_add_and_format(<4 x i32> inreg %arg) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
+; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 1, i32 1)
; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP8]], label %[[TMP9:.*]], label %[[BB11:.*]]
; IR: [[TMP9]]:
@@ -79,15 +78,16 @@ define amdgpu_cs void @atomic_add_and_format(<4 x i32> inreg %arg) {
;
; GCN-LABEL: atomic_add_and_format:
; GCN: ; %bb.0: ; %.entry
-; GCN-NEXT: s_mov_b64 s[6:7], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GCN-NEXT: s_mov_b64 s[4:5], exec
+; GCN-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN-NEXT: s_cbranch_execz .LBB1_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GCN-NEXT: s_mul_i32 s6, s6, 1
; GCN-NEXT: v_mov_b32_e32 v1, s6
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: buffer_atomic_add v1, v2, s[0:3], 0 idxen glc
@@ -119,8 +119,7 @@ define amdgpu_cs void @atomic_sub(<4 x i32> inreg %arg) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
+; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 1, i32 1)
; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP8]], label %[[BB9:.*]], label %[[BB11:.*]]
; IR: [[BB9]]:
@@ -131,14 +130,15 @@ define amdgpu_cs void @atomic_sub(<4 x i32> inreg %arg) {
;
; GCN-LABEL: atomic_sub:
; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
; GCN-NEXT: s_mov_b64 s[4:5], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s4, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s5, v0
+; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GCN-NEXT: s_cbranch_execz .LBB2_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
+; GCN-NEXT: s_mul_i32 s4, s4, 1
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: buffer_atomic_sub v0, v1, s[0:3], 0 idxen
@@ -159,8 +159,7 @@ define amdgpu_cs void @atomic_sub_and_format(<4 x i32> inreg %arg) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
+; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 1, i32 1)
; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP8]], label %[[TMP9:.*]], label %[[BB11:.*]]
; IR: [[TMP9]]:
@@ -175,15 +174,16 @@ define amdgpu_cs void @atomic_sub_and_format(<4 x i32> inreg %arg) {
;
; GCN-LABEL: atomic_sub_and_format:
; GCN: ; %bb.0: ; %.entry
-; GCN-NEXT: s_mov_b64 s[6:7], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GCN-NEXT: s_mov_b64 s[4:5], exec
+; GCN-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN-NEXT: s_cbranch_execz .LBB3_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GCN-NEXT: s_mul_i32 s6, s6, 1
; GCN-NEXT: v_mov_b32_e32 v1, s6
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: buffer_atomic_sub v1, v2, s[0:3], 0 idxen glc
@@ -215,9 +215,7 @@ define amdgpu_cs void @atomic_xor(<4 x i32> inreg %arg) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = and i32 [[TMP7]], 1
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.xor.i32(i32 1, i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label %[[BB10:.*]], label %[[BB12:.*]]
; IR: [[BB10]]:
@@ -228,15 +226,16 @@ define amdgpu_cs void @atomic_xor(<4 x i32> inreg %arg) {
;
; GCN-LABEL: atomic_xor:
; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
; GCN-NEXT: s_mov_b64 s[4:5], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s4, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s5, v0
+; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GCN-NEXT: s_cbranch_execz .LBB4_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GCN-NEXT: s_and_b32 s4, s4, 1
+; GCN-NEXT: s_mul_i32 s4, s4, 1
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: buffer_atomic_xor v0, v1, s[0:3], 0 idxen
@@ -257,9 +256,7 @@ define amdgpu_cs void @atomic_xor_and_format(<4 x i32> inreg %arg) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = and i32 [[TMP7]], 1
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.xor.i32(i32 1, i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label %[[TMP10:.*]], label %[[BB12:.*]]
; IR: [[TMP10]]:
@@ -275,16 +272,17 @@ define amdgpu_cs void @atomic_xor_and_format(<4 x i32> inreg %arg) {
;
; GCN-LABEL: atomic_xor_and_format:
; GCN: ; %bb.0: ; %.entry
-; GCN-NEXT: s_mov_b64 s[6:7], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GCN-NEXT: s_mov_b64 s[4:5], exec
+; GCN-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN-NEXT: s_cbranch_execz .LBB5_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GCN-NEXT: s_and_b32 s6, s6, 1
+; GCN-NEXT: s_mul_i32 s6, s6, 1
; GCN-NEXT: v_mov_b32_e32 v1, s6
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: buffer_atomic_xor v1, v2, s[0:3], 0 idxen glc
@@ -317,8 +315,7 @@ define amdgpu_cs void @atomic_ptr_add(ptr addrspace(8) inreg %arg) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
+; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 1, i32 1)
; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP8]], label %[[BB9:.*]], label %[[BB11:.*]]
; IR: [[BB9]]:
@@ -329,14 +326,15 @@ define amdgpu_cs void @atomic_ptr_add(ptr addrspace(8) inreg %arg) {
;
; GCN-LABEL: atomic_ptr_add:
; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
; GCN-NEXT: s_mov_b64 s[4:5], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s4, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s5, v0
+; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GCN-NEXT: s_cbranch_execz .LBB6_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
+; GCN-NEXT: s_mul_i32 s4, s4, 1
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: buffer_atomic_add v0, v1, s[0:3], 0 idxen
@@ -357,8 +355,7 @@ define amdgpu_cs void @atomic_ptr_add_and_format(ptr addrspace(8) inreg %arg) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
+; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 1, i32 1)
; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP8]], label %[[TMP9:.*]], label %[[BB11:.*]]
; IR: [[TMP9]]:
@@ -375,15 +372,16 @@ define amdgpu_cs void @atomic_ptr_add_and_format(ptr addrspace(8) inreg %arg) {
;
; GCN-LABEL: atomic_ptr_add_and_format:
; GCN: ; %bb.0: ; %.entry
-; GCN-NEXT: s_mov_b64 s[6:7], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GCN-NEXT: s_mov_b64 s[4:5], exec
+; GCN-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN-NEXT: s_cbranch_execz .LBB7_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GCN-NEXT: s_mul_i32 s6, s6, 1
; GCN-NEXT: v_mov_b32_e32 v1, s6
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: buffer_atomic_add v1, v2, s[0:3], 0 idxen glc
@@ -417,8 +415,7 @@ define amdgpu_cs void @atomic_ptr_sub(ptr addrspace(8) inreg %arg) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
+; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 1, i32 1)
; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP8]], label %[[BB9:.*]], label %[[BB11:.*]]
; IR: [[BB9]]:
@@ -429,14 +426,15 @@ define amdgpu_cs void @atomic_ptr_sub(ptr addrspace(8) inreg %arg) {
;
; GCN-LABEL: atomic_ptr_sub:
; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
; GCN-NEXT: s_mov_b64 s[4:5], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s4, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s5, v0
+; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GCN-NEXT: s_cbranch_execz .LBB8_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
+; GCN-NEXT: s_mul_i32 s4, s4, 1
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: buffer_atomic_sub v0, v1, s[0:3], 0 idxen
@@ -457,8 +455,7 @@ define amdgpu_cs void @atomic_ptr_sub_and_format(ptr addrspace(8) inreg %arg) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
+; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 1, i32 1)
; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP8]], label %[[TMP9:.*]], label %[[BB11:.*]]
; IR: [[TMP9]]:
@@ -475,15 +472,16 @@ define amdgpu_cs void @atomic_ptr_sub_and_format(ptr addrspace(8) inreg %arg) {
;
; GCN-LABEL: atomic_ptr_sub_and_format:
; GCN: ; %bb.0: ; %.entry
-; GCN-NEXT: s_mov_b64 s[6:7], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GCN-NEXT: s_mov_b64 s[4:5], exec
+; GCN-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN-NEXT: s_cbranch_execz .LBB9_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GCN-NEXT: s_mul_i32 s6, s6, 1
; GCN-NEXT: v_mov_b32_e32 v1, s6
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: buffer_atomic_sub v1, v2, s[0:3], 0 idxen glc
@@ -517,9 +515,7 @@ define amdgpu_cs void @atomic_ptr_xor(ptr addrspace(8) inreg %arg) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = and i32 [[TMP7]], 1
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.xor.i32(i32 1, i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label %[[BB10:.*]], label %[[BB12:.*]]
; IR: [[BB10]]:
@@ -530,15 +526,16 @@ define amdgpu_cs void @atomic_ptr_xor(ptr addrspace(8) inreg %arg) {
;
; GCN-LABEL: atomic_ptr_xor:
; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
; GCN-NEXT: s_mov_b64 s[4:5], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s4, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s5, v0
+; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GCN-NEXT: s_cbranch_execz .LBB10_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GCN-NEXT: s_and_b32 s4, s4, 1
+; GCN-NEXT: s_mul_i32 s4, s4, 1
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: buffer_atomic_xor v0, v1, s[0:3], 0 idxen
@@ -559,9 +556,7 @@ define amdgpu_cs void @atomic_ptr_xor_and_format(ptr addrspace(8) inreg %arg) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = and i32 [[TMP7]], 1
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.xor.i32(i32 1, i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label %[[TMP10:.*]], label %[[BB12:.*]]
; IR: [[TMP10]]:
@@ -579,16 +574,17 @@ define amdgpu_cs void @atomic_ptr_xor_and_format(ptr addrspace(8) inreg %arg) {
;
; GCN-LABEL: atomic_ptr_xor_and_format:
; GCN: ; %bb.0: ; %.entry
-; GCN-NEXT: s_mov_b64 s[6:7], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GCN-NEXT: s_mov_b64 s[4:5], exec
+; GCN-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN-NEXT: s_cbranch_execz .LBB11_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GCN-NEXT: s_and_b32 s6, s6, 1
+; GCN-NEXT: s_mul_i32 s6, s6, 1
; GCN-NEXT: v_mov_b32_e32 v1, s6
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: buffer_atomic_xor v1, v2, s[0:3], 0 idxen glc
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll
index 2f31b47d88ab0..6a55c98b37be1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll
@@ -1550,16 +1550,19 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat(ptr addrspace(1) %pt
; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat:
; GFX90A: ; %bb.0: ; %main_body
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
-; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX90A-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX90A-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX90A-NEXT: v_readfirstlane_b32 s0, v0
+; GFX90A-NEXT: v_readfirstlane_b32 s1, v1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB36_2
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1573,16 +1576,19 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat(ptr addrspace(1) %pt
; GFX942-LABEL: global_atomic_fadd_f64_noret_pat:
; GFX942: ; %bb.0: ; %main_body
; GFX942-NEXT: s_mov_b64 s[0:1], exec
-; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX942-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX942-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX942-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX942-NEXT: v_readfirstlane_b32 s0, v0
+; GFX942-NEXT: v_readfirstlane_b32 s1, v1
; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-NEXT: s_cbranch_execz .LBB36_2
; GFX942-NEXT: ; %bb.1:
; GFX942-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX942-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX942-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0
+; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-NEXT: v_mov_b32_e32 v2, 0
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1598,22 +1604,26 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat(ptr addrspace(1) %pt
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-NEXT: s_mov_b32 s1, exec_lo
-; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1250-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1250-NEXT: s_mov_b32 s2, exec_lo
+; GFX1250-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_mul_f64_e32 v[0:1], 4.0, v[0:1]
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1250-NEXT: s_cbranch_execz .LBB36_2
; GFX1250-NEXT: ; %bb.1:
-; GFX1250-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
-; GFX1250-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_mul_f64_e32 v[0:1], 4.0, v[0:1]
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT: global_atomic_add_f64 v2, v[0:1], s[2:3] scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -1628,16 +1638,19 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent(ptr addrspace(
; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_agent:
; GFX90A: ; %bb.0: ; %main_body
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
-; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX90A-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX90A-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX90A-NEXT: v_readfirstlane_b32 s0, v0
+; GFX90A-NEXT: v_readfirstlane_b32 s1, v1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB37_2
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: global_atomic_add_f64 v2, v[0:1], s[2:3]
@@ -1649,16 +1662,19 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent(ptr addrspace(
; GFX942-LABEL: global_atomic_fadd_f64_noret_pat_agent:
; GFX942: ; %bb.0: ; %main_body
; GFX942-NEXT: s_mov_b64 s[0:1], exec
-; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX942-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX942-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX942-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX942-NEXT: v_readfirstlane_b32 s0, v0
+; GFX942-NEXT: v_readfirstlane_b32 s1, v1
; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-NEXT: s_cbranch_execz .LBB37_2
; GFX942-NEXT: ; %bb.1:
; GFX942-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX942-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX942-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0
+; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-NEXT: v_mov_b32_e32 v2, 0
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1674,22 +1690,26 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent(ptr addrspace(
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-NEXT: s_mov_b32 s1, exec_lo
-; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1250-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1250-NEXT: s_mov_b32 s2, exec_lo
+; GFX1250-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_mul_f64_e32 v[0:1], 4.0, v[0:1]
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1250-NEXT: s_cbranch_execz .LBB37_2
; GFX1250-NEXT: ; %bb.1:
-; GFX1250-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
-; GFX1250-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_mul_f64_e32 v[0:1], 4.0, v[0:1]
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT: global_atomic_add_f64 v2, v[0:1], s[2:3] scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -1704,16 +1724,19 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_system(ptr addrspace
; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_system:
; GFX90A: ; %bb.0: ; %main_body
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
-; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX90A-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX90A-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX90A-NEXT: v_readfirstlane_b32 s0, v0
+; GFX90A-NEXT: v_readfirstlane_b32 s1, v1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB38_2
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1727,16 +1750,19 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_system(ptr addrspace
; GFX942-LABEL: global_atomic_fadd_f64_noret_pat_system:
; GFX942: ; %bb.0: ; %main_body
; GFX942-NEXT: s_mov_b64 s[0:1], exec
-; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX942-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX942-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX942-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX942-NEXT: v_readfirstlane_b32 s0, v0
+; GFX942-NEXT: v_readfirstlane_b32 s1, v1
; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-NEXT: s_cbranch_execz .LBB38_2
; GFX942-NEXT: ; %bb.1:
; GFX942-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX942-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX942-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0
+; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-NEXT: v_mov_b32_e32 v2, 0
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1752,22 +1778,26 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_system(ptr addrspace
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-NEXT: s_mov_b32 s1, exec_lo
-; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1250-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1250-NEXT: s_mov_b32 s2, exec_lo
+; GFX1250-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_mul_f64_e32 v[0:1], 4.0, v[0:1]
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1250-NEXT: s_cbranch_execz .LBB38_2
; GFX1250-NEXT: ; %bb.1:
-; GFX1250-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
-; GFX1250-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_mul_f64_e32 v[0:1], 4.0, v[0:1]
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT: global_atomic_add_f64 v2, v[0:1], s[2:3] scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -1782,16 +1812,19 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_flush(ptr addrspace(
; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_flush:
; GFX90A: ; %bb.0: ; %main_body
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
-; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX90A-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX90A-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX90A-NEXT: v_readfirstlane_b32 s0, v0
+; GFX90A-NEXT: v_readfirstlane_b32 s1, v1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB39_2
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: global_atomic_add_f64 v2, v[0:1], s[2:3]
@@ -1803,16 +1836,19 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_flush(ptr addrspace(
; GFX942-LABEL: global_atomic_fadd_f64_noret_pat_flush:
; GFX942: ; %bb.0: ; %main_body
; GFX942-NEXT: s_mov_b64 s[0:1], exec
-; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX942-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX942-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX942-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX942-NEXT: v_readfirstlane_b32 s0, v0
+; GFX942-NEXT: v_readfirstlane_b32 s1, v1
; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-NEXT: s_cbranch_execz .LBB39_2
; GFX942-NEXT: ; %bb.1:
; GFX942-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX942-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX942-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0
+; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-NEXT: v_mov_b32_e32 v2, 0
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1828,22 +1864,26 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_flush(ptr addrspace(
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-NEXT: s_mov_b32 s1, exec_lo
-; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1250-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1250-NEXT: s_mov_b32 s2, exec_lo
+; GFX1250-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_mul_f64_e32 v[0:1], 4.0, v[0:1]
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1250-NEXT: s_cbranch_execz .LBB39_2
; GFX1250-NEXT: ; %bb.1:
-; GFX1250-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
-; GFX1250-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_mul_f64_e32 v[0:1], 4.0, v[0:1]
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT: global_atomic_add_f64 v2, v[0:1], s[2:3] scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -1984,16 +2024,19 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent_safe(ptr addrs
; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_agent_safe:
; GFX90A: ; %bb.0: ; %main_body
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
-; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX90A-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX90A-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX90A-NEXT: v_readfirstlane_b32 s0, v0
+; GFX90A-NEXT: v_readfirstlane_b32 s1, v1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB43_2
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: global_atomic_add_f64 v2, v[0:1], s[2:3]
@@ -2005,16 +2048,19 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent_safe(ptr addrs
; GFX942-LABEL: global_atomic_fadd_f64_noret_pat_agent_safe:
; GFX942: ; %bb.0: ; %main_body
; GFX942-NEXT: s_mov_b64 s[0:1], exec
-; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX942-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX942-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX942-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX942-NEXT: v_readfirstlane_b32 s0, v0
+; GFX942-NEXT: v_readfirstlane_b32 s1, v1
; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-NEXT: s_cbranch_execz .LBB43_2
; GFX942-NEXT: ; %bb.1:
; GFX942-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX942-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX942-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0
+; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-NEXT: v_mov_b32_e32 v2, 0
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -2030,22 +2076,26 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent_safe(ptr addrs
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-NEXT: s_mov_b32 s1, exec_lo
-; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1250-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1250-NEXT: s_mov_b32 s2, exec_lo
+; GFX1250-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_mul_f64_e32 v[0:1], 4.0, v[0:1]
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1250-NEXT: s_cbranch_execz .LBB43_2
; GFX1250-NEXT: ; %bb.1:
-; GFX1250-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
-; GFX1250-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_mul_f64_e32 v[0:1], 4.0, v[0:1]
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT: global_atomic_add_f64 v2, v[0:1], s[2:3] scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -2382,16 +2432,19 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat(ptr addrspace(3) %ptr
; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat:
; GFX90A: ; %bb.0: ; %main_body
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
-; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX90A-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX90A-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX90A-NEXT: v_readfirstlane_b32 s0, v0
+; GFX90A-NEXT: v_readfirstlane_b32 s1, v1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB51_2
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_load_dword s2, s[4:5], 0x24
-; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v2, s2
; GFX90A-NEXT: ds_add_f64 v2, v[0:1]
@@ -2402,16 +2455,19 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat(ptr addrspace(3) %ptr
; GFX942-LABEL: local_atomic_fadd_f64_noret_pat:
; GFX942: ; %bb.0: ; %main_body
; GFX942-NEXT: s_mov_b64 s[0:1], exec
-; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX942-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX942-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX942-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX942-NEXT: v_readfirstlane_b32 s0, v0
+; GFX942-NEXT: v_readfirstlane_b32 s1, v1
; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-NEXT: s_cbranch_execz .LBB51_2
; GFX942-NEXT: ; %bb.1:
; GFX942-NEXT: s_load_dword s2, s[4:5], 0x24
-; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX942-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX942-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0
+; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v2, s2
; GFX942-NEXT: ds_add_f64 v2, v[0:1]
@@ -2425,18 +2481,23 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat(ptr addrspace(3) %ptr
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-NEXT: s_mov_b32 s1, exec_lo
-; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1250-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1250-NEXT: s_mov_b32 s2, exec_lo
+; GFX1250-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_mul_f64_e32 v[0:1], 4.0, v[0:1]
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1250-NEXT: s_cbranch_execz .LBB51_2
; GFX1250-NEXT: ; %bb.1:
-; GFX1250-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_dual_mul_f64 v[0:1], 4.0, v[0:1] :: v_dual_mov_b32 v2, s0
+; GFX1250-NEXT: v_mov_b32_e32 v2, s2
; GFX1250-NEXT: ds_add_f64 v2, v[0:1]
; GFX1250-NEXT: s_wait_dscnt 0x0
; GFX1250-NEXT: .LBB51_2:
@@ -2450,16 +2511,19 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush(ptr addrspace(3
; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat_flush:
; GFX90A: ; %bb.0: ; %main_body
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
-; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX90A-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX90A-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX90A-NEXT: v_readfirstlane_b32 s0, v0
+; GFX90A-NEXT: v_readfirstlane_b32 s1, v1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB52_2
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_load_dword s2, s[4:5], 0x24
-; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v2, s2
; GFX90A-NEXT: ds_add_f64 v2, v[0:1]
@@ -2470,16 +2534,19 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush(ptr addrspace(3
; GFX942-LABEL: local_atomic_fadd_f64_noret_pat_flush:
; GFX942: ; %bb.0: ; %main_body
; GFX942-NEXT: s_mov_b64 s[0:1], exec
-; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX942-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX942-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX942-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX942-NEXT: v_readfirstlane_b32 s0, v0
+; GFX942-NEXT: v_readfirstlane_b32 s1, v1
; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-NEXT: s_cbranch_execz .LBB52_2
; GFX942-NEXT: ; %bb.1:
; GFX942-NEXT: s_load_dword s2, s[4:5], 0x24
-; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX942-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX942-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0
+; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v2, s2
; GFX942-NEXT: ds_add_f64 v2, v[0:1]
@@ -2493,18 +2560,23 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush(ptr addrspace(3
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-NEXT: s_mov_b32 s1, exec_lo
-; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1250-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1250-NEXT: s_mov_b32 s2, exec_lo
+; GFX1250-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_mul_f64_e32 v[0:1], 4.0, v[0:1]
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1250-NEXT: s_cbranch_execz .LBB52_2
; GFX1250-NEXT: ; %bb.1:
-; GFX1250-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_dual_mul_f64 v[0:1], 4.0, v[0:1] :: v_dual_mov_b32 v2, s0
+; GFX1250-NEXT: v_mov_b32_e32 v2, s2
; GFX1250-NEXT: ds_add_f64 v2, v[0:1]
; GFX1250-NEXT: s_wait_dscnt 0x0
; GFX1250-NEXT: .LBB52_2:
@@ -2518,16 +2590,19 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush_safe(ptr addrsp
; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat_flush_safe:
; GFX90A: ; %bb.0: ; %main_body
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
-; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX90A-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX90A-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX90A-NEXT: v_readfirstlane_b32 s0, v0
+; GFX90A-NEXT: v_readfirstlane_b32 s1, v1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB53_2
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_load_dword s2, s[4:5], 0x24
-; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v2, s2
; GFX90A-NEXT: ds_add_f64 v2, v[0:1]
@@ -2538,16 +2613,19 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush_safe(ptr addrsp
; GFX942-LABEL: local_atomic_fadd_f64_noret_pat_flush_safe:
; GFX942: ; %bb.0: ; %main_body
; GFX942-NEXT: s_mov_b64 s[0:1], exec
-; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX942-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX942-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX942-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX942-NEXT: v_readfirstlane_b32 s0, v0
+; GFX942-NEXT: v_readfirstlane_b32 s1, v1
; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-NEXT: s_cbranch_execz .LBB53_2
; GFX942-NEXT: ; %bb.1:
; GFX942-NEXT: s_load_dword s2, s[4:5], 0x24
-; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX942-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX942-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0
+; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v2, s2
; GFX942-NEXT: ds_add_f64 v2, v[0:1]
@@ -2561,18 +2639,23 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush_safe(ptr addrsp
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-NEXT: s_mov_b32 s1, exec_lo
-; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1250-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1250-NEXT: s_mov_b32 s2, exec_lo
+; GFX1250-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_mul_f64_e32 v[0:1], 4.0, v[0:1]
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1250-NEXT: s_cbranch_execz .LBB53_2
; GFX1250-NEXT: ; %bb.1:
-; GFX1250-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_dual_mul_f64 v[0:1], 4.0, v[0:1] :: v_dual_mov_b32 v2, s0
+; GFX1250-NEXT: v_mov_b32_e32 v2, s2
; GFX1250-NEXT: ds_add_f64 v2, v[0:1]
; GFX1250-NEXT: s_wait_dscnt 0x0
; GFX1250-NEXT: .LBB53_2:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/global-atomic-fadd.f32-no-rtn.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/global-atomic-fadd.f32-no-rtn.ll
index 397354d93f95a..bd05809f67fbb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/global-atomic-fadd.f32-no-rtn.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/global-atomic-fadd.f32-no-rtn.ll
@@ -80,50 +80,26 @@ define amdgpu_ps void @global_atomic_fadd_f32_saddr_no_rtn_atomicrmw(ptr addrspa
; GFX908-NEXT: [[V_MBCNT_LO_U32_B32_e64_:%[0-9]+]]:vgpr_32 = V_MBCNT_LO_U32_B32_e64 [[COPY7]], [[COPY8]], implicit $exec
; GFX908-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
; GFX908-NEXT: [[V_MBCNT_HI_U32_B32_e64_:%[0-9]+]]:vgpr_32 = V_MBCNT_HI_U32_B32_e64 [[COPY9]], [[V_MBCNT_LO_U32_B32_e64_]], implicit $exec
- ; GFX908-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 2147483648
- ; GFX908-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX908-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
- ; GFX908-NEXT: [[V_SET_INACTIVE_B32_:%[0-9]+]]:vgpr_32 = V_SET_INACTIVE_B32 0, [[COPY2]], 0, [[COPY10]], [[DEF]], implicit $exec
- ; GFX908-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX908-NEXT: [[V_MOV_B32_dpp:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY11]], [[V_SET_INACTIVE_B32_]], 273, 15, 15, 0, implicit $exec
- ; GFX908-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_SET_INACTIVE_B32_]], 0, [[V_MOV_B32_dpp]], 0, 0, implicit $mode, implicit $exec
- ; GFX908-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX908-NEXT: [[V_MOV_B32_dpp1:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY12]], [[V_ADD_F32_e64_]], 274, 15, 15, 0, implicit $exec
- ; GFX908-NEXT: [[V_ADD_F32_e64_1:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_MOV_B32_dpp1]], 0, 0, implicit $mode, implicit $exec
- ; GFX908-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX908-NEXT: [[V_MOV_B32_dpp2:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY13]], [[V_ADD_F32_e64_1]], 276, 15, 15, 0, implicit $exec
- ; GFX908-NEXT: [[V_ADD_F32_e64_2:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_1]], 0, [[V_MOV_B32_dpp2]], 0, 0, implicit $mode, implicit $exec
- ; GFX908-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX908-NEXT: [[V_MOV_B32_dpp3:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY14]], [[V_ADD_F32_e64_2]], 280, 15, 15, 0, implicit $exec
- ; GFX908-NEXT: [[V_ADD_F32_e64_3:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_2]], 0, [[V_MOV_B32_dpp3]], 0, 0, implicit $mode, implicit $exec
- ; GFX908-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX908-NEXT: [[V_MOV_B32_dpp4:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY15]], [[V_ADD_F32_e64_3]], 322, 10, 15, 0, implicit $exec
- ; GFX908-NEXT: [[V_ADD_F32_e64_4:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_3]], 0, [[V_MOV_B32_dpp4]], 0, 0, implicit $mode, implicit $exec
- ; GFX908-NEXT: [[COPY16:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX908-NEXT: [[V_MOV_B32_dpp5:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY16]], [[V_ADD_F32_e64_4]], 323, 12, 15, 0, implicit $exec
- ; GFX908-NEXT: [[V_ADD_F32_e64_5:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_4]], 0, [[V_MOV_B32_dpp5]], 0, 0, implicit $mode, implicit $exec
- ; GFX908-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 63
- ; GFX908-NEXT: [[V_READLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READLANE_B32 [[V_ADD_F32_e64_5]], [[S_MOV_B32_2]]
- ; GFX908-NEXT: early-clobber %33:sreg_32 = STRICT_WWM [[V_READLANE_B32_]], implicit $exec
- ; GFX908-NEXT: [[COPY17:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GFX908-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 [[V_MBCNT_HI_U32_B32_e64_]], [[COPY17]], implicit $exec
- ; GFX908-NEXT: [[COPY18:%[0-9]+]]:sreg_64_xexec = COPY [[V_CMP_EQ_U32_e64_]]
- ; GFX908-NEXT: [[SI_IF1:%[0-9]+]]:sreg_64_xexec = SI_IF [[COPY18]], %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX908-NEXT: [[WAVE_REDUCE_FADD_PSEUDO_F32_:%[0-9]+]]:sgpr_32 = WAVE_REDUCE_FADD_PSEUDO_F32 [[COPY2]], 2, implicit $exec
+ ; GFX908-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[WAVE_REDUCE_FADD_PSEUDO_F32_]]
+ ; GFX908-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX908-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 [[V_MBCNT_HI_U32_B32_e64_]], [[COPY11]], implicit $exec
+ ; GFX908-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec = COPY [[V_CMP_EQ_U32_e64_]]
+ ; GFX908-NEXT: [[SI_IF1:%[0-9]+]]:sreg_64_xexec = SI_IF [[COPY12]], %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX908-NEXT: S_BRANCH %bb.3
; GFX908-NEXT: {{ $}}
- ; GFX908-NEXT: bb.3 (%ir-block.31):
+ ; GFX908-NEXT: bb.3 (%ir-block.17):
; GFX908-NEXT: successors: %bb.4(0x80000000)
; GFX908-NEXT: {{ $}}
- ; GFX908-NEXT: [[COPY19:%[0-9]+]]:vgpr_32 = COPY %33
; GFX908-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; GFX908-NEXT: GLOBAL_ATOMIC_ADD_F32_SADDR [[V_MOV_B32_e32_]], [[COPY19]], [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load store syncscope("wavefront") monotonic (f32) on %ir.ptr, addrspace 1)
+ ; GFX908-NEXT: GLOBAL_ATOMIC_ADD_F32_SADDR [[V_MOV_B32_e32_]], [[COPY10]], [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load store syncscope("wavefront") monotonic (f32) on %ir.ptr, addrspace 1)
; GFX908-NEXT: {{ $}}
; GFX908-NEXT: bb.4.Flow:
; GFX908-NEXT: successors: %bb.5(0x80000000)
; GFX908-NEXT: {{ $}}
; GFX908-NEXT: SI_END_CF [[SI_IF1]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX908-NEXT: {{ $}}
- ; GFX908-NEXT: bb.5 (%ir-block.33):
+ ; GFX908-NEXT: bb.5 (%ir-block.19):
; GFX908-NEXT: SI_END_CF [[SI_IF]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX908-NEXT: S_ENDPGM 0
;
@@ -153,50 +129,26 @@ define amdgpu_ps void @global_atomic_fadd_f32_saddr_no_rtn_atomicrmw(ptr addrspa
; GFX90A-NEXT: [[V_MBCNT_LO_U32_B32_e64_:%[0-9]+]]:vgpr_32 = V_MBCNT_LO_U32_B32_e64 [[COPY7]], [[COPY8]], implicit $exec
; GFX90A-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
; GFX90A-NEXT: [[V_MBCNT_HI_U32_B32_e64_:%[0-9]+]]:vgpr_32 = V_MBCNT_HI_U32_B32_e64 [[COPY9]], [[V_MBCNT_LO_U32_B32_e64_]], implicit $exec
- ; GFX90A-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 2147483648
- ; GFX90A-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX90A-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
- ; GFX90A-NEXT: [[V_SET_INACTIVE_B32_:%[0-9]+]]:vgpr_32 = V_SET_INACTIVE_B32 0, [[COPY2]], 0, [[COPY10]], [[DEF]], implicit $exec
- ; GFX90A-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX90A-NEXT: [[V_MOV_B32_dpp:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY11]], [[V_SET_INACTIVE_B32_]], 273, 15, 15, 0, implicit $exec
- ; GFX90A-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_SET_INACTIVE_B32_]], 0, [[V_MOV_B32_dpp]], 0, 0, implicit $mode, implicit $exec
- ; GFX90A-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX90A-NEXT: [[V_MOV_B32_dpp1:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY12]], [[V_ADD_F32_e64_]], 274, 15, 15, 0, implicit $exec
- ; GFX90A-NEXT: [[V_ADD_F32_e64_1:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_MOV_B32_dpp1]], 0, 0, implicit $mode, implicit $exec
- ; GFX90A-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX90A-NEXT: [[V_MOV_B32_dpp2:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY13]], [[V_ADD_F32_e64_1]], 276, 15, 15, 0, implicit $exec
- ; GFX90A-NEXT: [[V_ADD_F32_e64_2:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_1]], 0, [[V_MOV_B32_dpp2]], 0, 0, implicit $mode, implicit $exec
- ; GFX90A-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX90A-NEXT: [[V_MOV_B32_dpp3:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY14]], [[V_ADD_F32_e64_2]], 280, 15, 15, 0, implicit $exec
- ; GFX90A-NEXT: [[V_ADD_F32_e64_3:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_2]], 0, [[V_MOV_B32_dpp3]], 0, 0, implicit $mode, implicit $exec
- ; GFX90A-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX90A-NEXT: [[V_MOV_B32_dpp4:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY15]], [[V_ADD_F32_e64_3]], 322, 10, 15, 0, implicit $exec
- ; GFX90A-NEXT: [[V_ADD_F32_e64_4:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_3]], 0, [[V_MOV_B32_dpp4]], 0, 0, implicit $mode, implicit $exec
- ; GFX90A-NEXT: [[COPY16:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX90A-NEXT: [[V_MOV_B32_dpp5:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY16]], [[V_ADD_F32_e64_4]], 323, 12, 15, 0, implicit $exec
- ; GFX90A-NEXT: [[V_ADD_F32_e64_5:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_4]], 0, [[V_MOV_B32_dpp5]], 0, 0, implicit $mode, implicit $exec
- ; GFX90A-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 63
- ; GFX90A-NEXT: [[V_READLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READLANE_B32 [[V_ADD_F32_e64_5]], [[S_MOV_B32_2]]
- ; GFX90A-NEXT: early-clobber %33:sreg_32 = STRICT_WWM [[V_READLANE_B32_]], implicit $exec
- ; GFX90A-NEXT: [[COPY17:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GFX90A-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 [[V_MBCNT_HI_U32_B32_e64_]], [[COPY17]], implicit $exec
- ; GFX90A-NEXT: [[COPY18:%[0-9]+]]:sreg_64_xexec = COPY [[V_CMP_EQ_U32_e64_]]
- ; GFX90A-NEXT: [[SI_IF1:%[0-9]+]]:sreg_64_xexec = SI_IF [[COPY18]], %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX90A-NEXT: [[WAVE_REDUCE_FADD_PSEUDO_F32_:%[0-9]+]]:sgpr_32 = WAVE_REDUCE_FADD_PSEUDO_F32 [[COPY2]], 2, implicit $exec
+ ; GFX90A-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[WAVE_REDUCE_FADD_PSEUDO_F32_]]
+ ; GFX90A-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX90A-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 [[V_MBCNT_HI_U32_B32_e64_]], [[COPY11]], implicit $exec
+ ; GFX90A-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec = COPY [[V_CMP_EQ_U32_e64_]]
+ ; GFX90A-NEXT: [[SI_IF1:%[0-9]+]]:sreg_64_xexec = SI_IF [[COPY12]], %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: S_BRANCH %bb.3
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: bb.3 (%ir-block.31):
+ ; GFX90A-NEXT: bb.3 (%ir-block.17):
; GFX90A-NEXT: successors: %bb.4(0x80000000)
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: [[COPY19:%[0-9]+]]:vgpr_32 = COPY %33
; GFX90A-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; GFX90A-NEXT: GLOBAL_ATOMIC_ADD_F32_SADDR [[V_MOV_B32_e32_]], [[COPY19]], [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load store syncscope("wavefront") monotonic (f32) on %ir.ptr, addrspace 1)
+ ; GFX90A-NEXT: GLOBAL_ATOMIC_ADD_F32_SADDR [[V_MOV_B32_e32_]], [[COPY10]], [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load store syncscope("wavefront") monotonic (f32) on %ir.ptr, addrspace 1)
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.4.Flow:
; GFX90A-NEXT: successors: %bb.5(0x80000000)
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: SI_END_CF [[SI_IF1]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: bb.5 (%ir-block.33):
+ ; GFX90A-NEXT: bb.5 (%ir-block.19):
; GFX90A-NEXT: SI_END_CF [[SI_IF]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: S_ENDPGM 0
;
@@ -226,50 +178,26 @@ define amdgpu_ps void @global_atomic_fadd_f32_saddr_no_rtn_atomicrmw(ptr addrspa
; GFX942-NEXT: [[V_MBCNT_LO_U32_B32_e64_:%[0-9]+]]:vgpr_32 = V_MBCNT_LO_U32_B32_e64 [[COPY7]], [[COPY8]], implicit $exec
; GFX942-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
; GFX942-NEXT: [[V_MBCNT_HI_U32_B32_e64_:%[0-9]+]]:vgpr_32 = V_MBCNT_HI_U32_B32_e64 [[COPY9]], [[V_MBCNT_LO_U32_B32_e64_]], implicit $exec
- ; GFX942-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 2147483648
- ; GFX942-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX942-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
- ; GFX942-NEXT: [[V_SET_INACTIVE_B32_:%[0-9]+]]:vgpr_32 = V_SET_INACTIVE_B32 0, [[COPY2]], 0, [[COPY10]], [[DEF]], implicit $exec
- ; GFX942-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX942-NEXT: [[V_MOV_B32_dpp:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY11]], [[V_SET_INACTIVE_B32_]], 273, 15, 15, 0, implicit $exec
- ; GFX942-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_SET_INACTIVE_B32_]], 0, [[V_MOV_B32_dpp]], 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX942-NEXT: [[V_MOV_B32_dpp1:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY12]], [[V_ADD_F32_e64_]], 274, 15, 15, 0, implicit $exec
- ; GFX942-NEXT: [[V_ADD_F32_e64_1:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_MOV_B32_dpp1]], 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX942-NEXT: [[V_MOV_B32_dpp2:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY13]], [[V_ADD_F32_e64_1]], 276, 15, 15, 0, implicit $exec
- ; GFX942-NEXT: [[V_ADD_F32_e64_2:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_1]], 0, [[V_MOV_B32_dpp2]], 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX942-NEXT: [[V_MOV_B32_dpp3:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY14]], [[V_ADD_F32_e64_2]], 280, 15, 15, 0, implicit $exec
- ; GFX942-NEXT: [[V_ADD_F32_e64_3:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_2]], 0, [[V_MOV_B32_dpp3]], 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX942-NEXT: [[V_MOV_B32_dpp4:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY15]], [[V_ADD_F32_e64_3]], 322, 10, 15, 0, implicit $exec
- ; GFX942-NEXT: [[V_ADD_F32_e64_4:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_3]], 0, [[V_MOV_B32_dpp4]], 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: [[COPY16:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX942-NEXT: [[V_MOV_B32_dpp5:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY16]], [[V_ADD_F32_e64_4]], 323, 12, 15, 0, implicit $exec
- ; GFX942-NEXT: [[V_ADD_F32_e64_5:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_4]], 0, [[V_MOV_B32_dpp5]], 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 63
- ; GFX942-NEXT: [[V_READLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READLANE_B32 [[V_ADD_F32_e64_5]], [[S_MOV_B32_2]]
- ; GFX942-NEXT: early-clobber %32:sreg_32 = STRICT_WWM [[V_READLANE_B32_]], implicit $exec
- ; GFX942-NEXT: [[COPY17:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GFX942-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 [[V_MBCNT_HI_U32_B32_e64_]], [[COPY17]], implicit $exec
- ; GFX942-NEXT: [[COPY18:%[0-9]+]]:sreg_64_xexec = COPY [[V_CMP_EQ_U32_e64_]]
- ; GFX942-NEXT: [[SI_IF1:%[0-9]+]]:sreg_64_xexec = SI_IF [[COPY18]], %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX942-NEXT: [[WAVE_REDUCE_FADD_PSEUDO_F32_:%[0-9]+]]:sgpr_32 = WAVE_REDUCE_FADD_PSEUDO_F32 [[COPY2]], 2, implicit $exec
+ ; GFX942-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[WAVE_REDUCE_FADD_PSEUDO_F32_]]
+ ; GFX942-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX942-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 [[V_MBCNT_HI_U32_B32_e64_]], [[COPY11]], implicit $exec
+ ; GFX942-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec = COPY [[V_CMP_EQ_U32_e64_]]
+ ; GFX942-NEXT: [[SI_IF1:%[0-9]+]]:sreg_64_xexec = SI_IF [[COPY12]], %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX942-NEXT: S_BRANCH %bb.3
; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: bb.3 (%ir-block.31):
+ ; GFX942-NEXT: bb.3 (%ir-block.17):
; GFX942-NEXT: successors: %bb.4(0x80000000)
; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: [[COPY19:%[0-9]+]]:vgpr_32 = COPY %32
; GFX942-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: GLOBAL_ATOMIC_ADD_F32_SADDR [[V_MOV_B32_e32_]], [[COPY19]], [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load store syncscope("wavefront") monotonic (f32) on %ir.ptr, addrspace 1)
+ ; GFX942-NEXT: GLOBAL_ATOMIC_ADD_F32_SADDR [[V_MOV_B32_e32_]], [[COPY10]], [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load store syncscope("wavefront") monotonic (f32) on %ir.ptr, addrspace 1)
; GFX942-NEXT: {{ $}}
; GFX942-NEXT: bb.4.Flow:
; GFX942-NEXT: successors: %bb.5(0x80000000)
; GFX942-NEXT: {{ $}}
; GFX942-NEXT: SI_END_CF [[SI_IF1]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: bb.5 (%ir-block.33):
+ ; GFX942-NEXT: bb.5 (%ir-block.19):
; GFX942-NEXT: SI_END_CF [[SI_IF]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX942-NEXT: S_ENDPGM 0
;
@@ -282,7 +210,6 @@ define amdgpu_ps void @global_atomic_fadd_f32_saddr_no_rtn_atomicrmw(ptr addrspa
; GFX11-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64_xexec_xnull = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX11-NEXT: [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
; GFX11-NEXT: [[SI_PS_LIVE:%[0-9]+]]:sreg_32_xm0_xexec = SI_PS_LIVE
; GFX11-NEXT: [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec = SI_IF [[SI_PS_LIVE]], %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX11-NEXT: S_BRANCH %bb.2
@@ -295,44 +222,26 @@ define amdgpu_ps void @global_atomic_fadd_f32_saddr_no_rtn_atomicrmw(ptr addrspa
; GFX11-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[COPY3]]
; GFX11-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
; GFX11-NEXT: [[V_MBCNT_LO_U32_B32_e64_:%[0-9]+]]:vgpr_32 = V_MBCNT_LO_U32_B32_e64 [[COPY4]], [[COPY5]], implicit $exec
- ; GFX11-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 2147483648
- ; GFX11-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX11-NEXT: [[DEF1:%[0-9]+]]:sreg_32_xm0_xexec = IMPLICIT_DEF
- ; GFX11-NEXT: [[V_SET_INACTIVE_B32_:%[0-9]+]]:vgpr_32 = V_SET_INACTIVE_B32 0, [[COPY2]], 0, [[COPY6]], [[DEF1]], implicit $exec
- ; GFX11-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX11-NEXT: [[V_MOV_B32_dpp:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY7]], [[V_SET_INACTIVE_B32_]], 353, 15, 15, 0, implicit $exec
- ; GFX11-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_SET_INACTIVE_B32_]], 0, [[V_MOV_B32_dpp]], 0, 0, implicit $mode, implicit $exec
- ; GFX11-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX11-NEXT: [[V_MOV_B32_dpp1:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY8]], [[V_ADD_F32_e64_]], 354, 15, 15, 0, implicit $exec
- ; GFX11-NEXT: [[V_ADD_F32_e64_1:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_MOV_B32_dpp1]], 0, 0, implicit $mode, implicit $exec
- ; GFX11-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX11-NEXT: [[V_MOV_B32_dpp2:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY9]], [[V_ADD_F32_e64_1]], 356, 15, 15, 0, implicit $exec
- ; GFX11-NEXT: [[V_ADD_F32_e64_2:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_1]], 0, [[V_MOV_B32_dpp2]], 0, 0, implicit $mode, implicit $exec
- ; GFX11-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX11-NEXT: [[V_MOV_B32_dpp3:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY10]], [[V_ADD_F32_e64_2]], 360, 15, 15, 0, implicit $exec
- ; GFX11-NEXT: [[V_ADD_F32_e64_3:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_2]], 0, [[V_MOV_B32_dpp3]], 0, 0, implicit $mode, implicit $exec
- ; GFX11-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[DEF]]
- ; GFX11-NEXT: [[V_PERMLANEX16_B32_e64_:%[0-9]+]]:vgpr_32 = V_PERMLANEX16_B32_e64 0, [[V_ADD_F32_e64_3]], 0, [[S_MOV_B32_]], 0, [[S_MOV_B32_]], [[COPY11]], 0, implicit $exec
- ; GFX11-NEXT: [[V_ADD_F32_e64_4:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_3]], 0, [[V_PERMLANEX16_B32_e64_]], 0, 0, implicit $mode, implicit $exec
- ; GFX11-NEXT: early-clobber %24:vgpr_32 = STRICT_WWM [[V_ADD_F32_e64_4]], implicit $exec
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GFX11-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_EQ_U32_e64 [[V_MBCNT_LO_U32_B32_e64_]], [[COPY12]], implicit $exec
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:sreg_32_xm0_xexec = COPY [[V_CMP_EQ_U32_e64_]]
- ; GFX11-NEXT: [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec = SI_IF [[COPY13]], %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX11-NEXT: [[WAVE_REDUCE_FADD_PSEUDO_F32_:%[0-9]+]]:sgpr_32 = WAVE_REDUCE_FADD_PSEUDO_F32 [[COPY2]], 2, implicit $exec
+ ; GFX11-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[WAVE_REDUCE_FADD_PSEUDO_F32_]]
+ ; GFX11-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX11-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_EQ_U32_e64 [[V_MBCNT_LO_U32_B32_e64_]], [[COPY7]], implicit $exec
+ ; GFX11-NEXT: [[COPY8:%[0-9]+]]:sreg_32_xm0_xexec = COPY [[V_CMP_EQ_U32_e64_]]
+ ; GFX11-NEXT: [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec = SI_IF [[COPY8]], %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX11-NEXT: S_BRANCH %bb.3
; GFX11-NEXT: {{ $}}
- ; GFX11-NEXT: bb.3 (%ir-block.24):
+ ; GFX11-NEXT: bb.3 (%ir-block.13):
; GFX11-NEXT: successors: %bb.4(0x80000000)
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; GFX11-NEXT: GLOBAL_ATOMIC_ADD_F32_SADDR [[V_MOV_B32_e32_]], %24, [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load store syncscope("wavefront") monotonic (f32) on %ir.ptr, addrspace 1)
+ ; GFX11-NEXT: GLOBAL_ATOMIC_ADD_F32_SADDR [[V_MOV_B32_e32_]], [[COPY6]], [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load store syncscope("wavefront") monotonic (f32) on %ir.ptr, addrspace 1)
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: bb.4.Flow:
; GFX11-NEXT: successors: %bb.5(0x80000000)
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: SI_END_CF [[SI_IF1]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX11-NEXT: {{ $}}
- ; GFX11-NEXT: bb.5 (%ir-block.26):
+ ; GFX11-NEXT: bb.5 (%ir-block.15):
; GFX11-NEXT: SI_END_CF [[SI_IF]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX11-NEXT: S_ENDPGM 0
%ret = atomicrmw fadd ptr addrspace(1) %ptr, float %data syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !0
diff --git a/llvm/test/CodeGen/AMDGPU/atomic-optimizer-promote-i8.ll b/llvm/test/CodeGen/AMDGPU/atomic-optimizer-promote-i8.ll
index d2ff3c25a1acb..ee1758316d717 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic-optimizer-promote-i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic-optimizer-promote-i8.ll
@@ -10,13 +10,14 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; CHECK-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)
; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])
+; CHECK-NEXT: [[TMP8:%.*]] = call i8 @llvm.amdgcn.wave.reduce.or.i8(i8 [[VAL]], i32 1)
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP6]], 0
; CHECK-NEXT: br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]
; CHECK: [[BB8]]:
-; CHECK-NEXT: [[TMP9:%.*]] = atomicrmw or ptr addrspace(1) [[UNIFORM_PTR]], i8 [[VAL]] monotonic, align 1
+; CHECK-NEXT: [[TMP10:%.*]] = atomicrmw or ptr addrspace(1) [[UNIFORM_PTR]], i8 [[TMP8]] monotonic, align 1
; CHECK-NEXT: br label %[[BB10]]
; CHECK: [[BB10]]:
-; CHECK-NEXT: [[TMP11:%.*]] = phi i8 [ poison, [[TMP0:%.*]] ], [ [[TMP9]], %[[BB8]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = phi i8 [ poison, [[TMP0:%.*]] ], [ [[TMP10]], %[[BB8]] ]
; CHECK-NEXT: [[TMP16:%.*]] = zext i8 [[TMP11]] to i32
; CHECK-NEXT: [[TMP17:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP16]])
; CHECK-NEXT: [[TMP12:%.*]] = trunc i32 [[TMP17]] to i8
@@ -40,9 +41,7 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; CHECK-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)
; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP1]])
-; CHECK-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i8
-; CHECK-NEXT: [[TMP9:%.*]] = mul i8 [[VAL]], [[TMP8]]
+; CHECK-NEXT: [[TMP9:%.*]] = call i8 @llvm.amdgcn.wave.reduce.add.i8(i8 [[VAL]], i32 1)
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP6]], 0
; CHECK-NEXT: br i1 [[TMP10]], label %[[BB11:.*]], label %[[BB13:.*]]
; CHECK: [[BB11]]:
@@ -85,13 +84,14 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; CHECK-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)
; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])
+; CHECK-NEXT: [[TMP8:%.*]] = call i16 @llvm.amdgcn.wave.reduce.or.i16(i16 [[VAL]], i32 1)
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP6]], 0
; CHECK-NEXT: br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]
; CHECK: [[BB8]]:
-; CHECK-NEXT: [[TMP9:%.*]] = atomicrmw or ptr addrspace(1) [[UNIFORM_PTR]], i16 [[VAL]] monotonic, align 2
+; CHECK-NEXT: [[TMP10:%.*]] = atomicrmw or ptr addrspace(1) [[UNIFORM_PTR]], i16 [[TMP8]] monotonic, align 2
; CHECK-NEXT: br label %[[BB10]]
; CHECK: [[BB10]]:
-; CHECK-NEXT: [[TMP11:%.*]] = phi i16 [ poison, [[TMP0:%.*]] ], [ [[TMP9]], %[[BB8]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = phi i16 [ poison, [[TMP0:%.*]] ], [ [[TMP10]], %[[BB8]] ]
; CHECK-NEXT: [[TMP16:%.*]] = zext i16 [[TMP11]] to i32
; CHECK-NEXT: [[TMP17:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP16]])
; CHECK-NEXT: [[TMP12:%.*]] = trunc i32 [[TMP17]] to i16
@@ -115,9 +115,7 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; CHECK-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)
; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP1]])
-; CHECK-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i16
-; CHECK-NEXT: [[TMP9:%.*]] = mul i16 [[VAL]], [[TMP8]]
+; CHECK-NEXT: [[TMP9:%.*]] = call i16 @llvm.amdgcn.wave.reduce.add.i16(i16 [[VAL]], i32 1)
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP6]], 0
; CHECK-NEXT: br i1 [[TMP10]], label %[[BB11:.*]], label %[[BB13:.*]]
; CHECK: [[BB11]]:
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimization_split_dt_update.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimization_split_dt_update.ll
index 9fafedcaba475..80348a9a79d46 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimization_split_dt_update.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimization_split_dt_update.ll
@@ -34,30 +34,16 @@ define amdgpu_kernel void @ham(ptr addrspace(4) %arg) {
; CHECK-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; CHECK-NEXT: br label [[COMPUTELOOP:%.*]]
-; CHECK: 7:
-; CHECK-NEXT: [[TMP8:%.*]] = atomicrmw add ptr addrspace(1) [[ADDRSPACECAST]], i32 [[TMP13:%.*]] syncscope("agent-one-as") monotonic, align 4
-; CHECK-NEXT: br label [[TMP9:%.*]]
-; CHECK: 9:
+; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[PHI]], i32 1)
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP5]], 0
+; CHECK-NEXT: br i1 [[TMP7]], label [[TMP10:%.*]], label [[TMP9:%.*]]
+; CHECK: 8:
+; CHECK-NEXT: [[TMP8:%.*]] = atomicrmw add ptr addrspace(1) [[ADDRSPACECAST]], i32 [[TMP13]] syncscope("agent-one-as") monotonic, align 4
+; CHECK-NEXT: br label [[TMP9]]
+; CHECK: 10:
; CHECK-NEXT: br label [[BB8]]
; CHECK: bb8:
; CHECK-NEXT: br label [[BB4]]
-; CHECK: ComputeLoop:
-; CHECK-NEXT: [[ACCUMULATOR:%.*]] = phi i32 [ 0, [[BB7]] ], [ [[TMP13]], [[COMPUTELOOP]] ]
-; CHECK-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP6]], [[BB7]] ], [ [[TMP16:%.*]], [[COMPUTELOOP]] ]
-; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true)
-; CHECK-NEXT: [[TMP11:%.*]] = trunc i64 [[TMP10]] to i32
-; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[PHI]], i32 [[TMP11]])
-; CHECK-NEXT: [[TMP13]] = add i32 [[ACCUMULATOR]], [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = shl i64 1, [[TMP10]]
-; CHECK-NEXT: [[TMP15:%.*]] = xor i64 [[TMP14]], -1
-; CHECK-NEXT: [[TMP16]] = and i64 [[ACTIVEBITS]], [[TMP15]]
-; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[TMP16]], 0
-; CHECK-NEXT: br i1 [[TMP17]], label [[COMPUTEEND:%.*]], label [[COMPUTELOOP]]
-; CHECK: ComputeEnd:
-; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i32 [[TMP5]], 0
-; CHECK-NEXT: br i1 [[TMP18]], label [[TMP7:%.*]], label [[TMP9]]
;
bb:
%call = tail call i32 @quux()
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
index ebec9537f3f1a..6cf7b2c22cf59 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
@@ -21,16 +21,16 @@ declare i32 @llvm.amdgcn.raw.ptr.buffer.atomic.sub(i32, ptr addrspace(8), i32, i
define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace(8) %inout) {
; GFX6-LABEL: add_i32_constant:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_mov_b64 s[2:3], exec
-; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX6-NEXT: s_mov_b64 s[0:1], exec
+; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX6-NEXT: ; implicit-def: $vgpr1
; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX6-NEXT: s_cbranch_execz .LBB0_2
; GFX6-NEXT: ; %bb.1:
; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd
-; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX6-NEXT: s_mul_i32 s2, s2, 5
; GFX6-NEXT: v_mov_b32_e32 v1, s2
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
@@ -49,16 +49,16 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX8-LABEL: add_i32_constant:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB0_2
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX8-NEXT: s_mul_i32 s2, s2, 5
; GFX8-NEXT: v_mov_b32_e32 v1, s2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
@@ -77,16 +77,16 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX9-LABEL: add_i32_constant:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB0_2
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_mul_i32 s2, s2, 5
; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -104,16 +104,16 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX10W64-LABEL: add_i32_constant:
; GFX10W64: ; %bb.0: ; %entry
-; GFX10W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W64-NEXT: s_mov_b64 s[0:1], exec
; GFX10W64-NEXT: ; implicit-def: $vgpr1
-; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX10W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX10W64-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX10W64-NEXT: s_cbranch_execz .LBB0_2
; GFX10W64-NEXT: ; %bb.1:
; GFX10W64-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX10W64-NEXT: s_mul_i32 s2, s2, 5
; GFX10W64-NEXT: v_mov_b32_e32 v1, s2
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
@@ -132,15 +132,15 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX10W32-LABEL: add_i32_constant:
; GFX10W32: ; %bb.0: ; %entry
-; GFX10W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W32-NEXT: s_mov_b32 s0, exec_lo
; GFX10W32-NEXT: ; implicit-def: $vgpr1
-; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
+; GFX10W32-NEXT: s_bcnt1_i32_b32 s1, s0
; GFX10W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10W32-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX10W32-NEXT: s_cbranch_execz .LBB0_2
; GFX10W32-NEXT: ; %bb.1:
; GFX10W32-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W32-NEXT: s_bcnt1_i32_b32 s1, s1
; GFX10W32-NEXT: s_mul_i32 s1, s1, 5
; GFX10W32-NEXT: v_mov_b32_e32 v1, s1
; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)
@@ -159,19 +159,20 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX11W64-LABEL: add_i32_constant:
; GFX11W64: ; %bb.0: ; %entry
-; GFX11W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W64-NEXT: ; implicit-def: $vgpr1
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W64-NEXT: s_cbranch_execz .LBB0_2
; GFX11W64-NEXT: ; %bb.1:
; GFX11W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11W64-NEXT: s_mul_i32 s2, s2, 5
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11W64-NEXT: v_mov_b32_e32 v1, s2
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W64-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], 0 glc
@@ -189,18 +190,18 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX11W32-LABEL: add_i32_constant:
; GFX11W32: ; %bb.0: ; %entry
-; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX11W32-NEXT: ; implicit-def: $vgpr1
-; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX11W32-NEXT: s_mov_b32 s0, exec_lo
; GFX11W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W32-NEXT: s_cbranch_execz .LBB0_2
; GFX11W32-NEXT: ; %bb.1:
; GFX11W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11W32-NEXT: s_mul_i32 s1, s1, 5
+; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11W32-NEXT: v_mov_b32_e32 v1, s1
; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W32-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], 0 glc
@@ -218,20 +219,20 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX12W64-LABEL: add_i32_constant:
; GFX12W64: ; %bb.0: ; %entry
-; GFX12W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W64-NEXT: ; implicit-def: $vgpr1
-; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W64-NEXT: s_cbranch_execz .LBB0_2
; GFX12W64-NEXT: ; %bb.1:
; GFX12W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W64-NEXT: s_mul_i32 s2, s2, 5
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12W64-NEXT: v_mov_b32_e32 v1, s2
; GFX12W64-NEXT: s_wait_kmcnt 0x0
; GFX12W64-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -250,19 +251,18 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX12W32-LABEL: add_i32_constant:
; GFX12W32: ; %bb.0: ; %entry
-; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX12W32-NEXT: ; implicit-def: $vgpr1
-; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX12W32-NEXT: s_mov_b32 s0, exec_lo
; GFX12W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W32-NEXT: s_cbranch_execz .LBB0_2
; GFX12W32-NEXT: ; %bb.1:
; GFX12W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W32-NEXT: s_mul_i32 s1, s1, 5
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12W32-NEXT: v_mov_b32_e32 v1, s1
; GFX12W32-NEXT: s_wait_kmcnt 0x0
; GFX12W32-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -280,19 +280,20 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX13W64-LABEL: add_i32_constant:
; GFX13W64: ; %bb.0: ; %entry
-; GFX13W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W64-NEXT: ; implicit-def: $vgpr1
-; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX13W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W64-NEXT: s_cbranch_execz .LBB0_2
; GFX13W64-NEXT: ; %bb.1:
; GFX13W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX13W64-NEXT: s_mul_i32 s2, s2, 5
+; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX13W64-NEXT: v_mov_b32_e32 v1, s2
; GFX13W64-NEXT: s_wait_kmcnt 0x0
; GFX13W64-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -310,18 +311,18 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX13W32-LABEL: add_i32_constant:
; GFX13W32: ; %bb.0: ; %entry
-; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX13W32-NEXT: ; implicit-def: $vgpr1
-; GFX13W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX13W32-NEXT: s_mov_b32 s0, exec_lo
; GFX13W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W32-NEXT: s_cbranch_execz .LBB0_2
; GFX13W32-NEXT: ; %bb.1:
; GFX13W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX13W32-NEXT: s_mul_i32 s1, s1, 5
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX13W32-NEXT: v_mov_b32_e32 v1, s1
; GFX13W32-NEXT: s_wait_kmcnt 0x0
; GFX13W32-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -345,26 +346,26 @@ entry:
define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(8) %inout, i32 %additive) {
; GFX6-LABEL: add_i32_uniform:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0x11
-; GFX6-NEXT: s_mov_b64 s[2:3], exec
-; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX6-NEXT: s_load_dword s2, s[4:5], 0x11
+; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX6-NEXT: s_mov_b64 s[0:1], exec
+; GFX6-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX6-NEXT: ; implicit-def: $vgpr1
; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX6-NEXT: s_cbranch_execz .LBB1_2
; GFX6-NEXT: ; %bb.1:
; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd
-; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mul_i32 s2, s6, s2
-; GFX6-NEXT: v_mov_b32_e32 v1, s2
+; GFX6-NEXT: s_mul_i32 s3, s2, s3
+; GFX6-NEXT: v_mov_b32_e32 v1, s3
; GFX6-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
; GFX6-NEXT: .LBB1_2:
; GFX6-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX6-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_readfirstlane_b32 s4, v1
; GFX6-NEXT: s_mov_b32 s3, 0xf000
@@ -375,26 +376,26 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX8-LABEL: add_i32_uniform:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB1_2
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mul_i32 s2, s6, s2
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
+; GFX8-NEXT: s_mul_i32 s3, s2, s3
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
; GFX8-NEXT: .LBB1_2:
; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX8-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s2, v1
; GFX8-NEXT: v_add_u32_e32 v2, vcc, s2, v0
@@ -405,26 +406,26 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX9-LABEL: add_i32_uniform:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB1_2
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_i32 s2, s6, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: s_mul_i32 s3, s2, s3
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
; GFX9-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
; GFX9-NEXT: .LBB1_2:
; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX9-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s2, v1
; GFX9-NEXT: v_mov_b32_e32 v2, 0
@@ -434,29 +435,30 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX10W64-LABEL: add_i32_uniform:
; GFX10W64: ; %bb.0: ; %entry
-; GFX10W64-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX10W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX10W64-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W64-NEXT: s_mov_b64 s[0:1], exec
; GFX10W64-NEXT: ; implicit-def: $vgpr1
-; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX10W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX10W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX10W64-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX10W64-NEXT: s_cbranch_execz .LBB1_2
; GFX10W64-NEXT: ; %bb.1:
; GFX10W64-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10W64-NEXT: s_mul_i32 s2, s6, s2
-; GFX10W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX10W64-NEXT: s_mul_i32 s3, s2, s3
+; GFX10W64-NEXT: v_mov_b32_e32 v1, s3
; GFX10W64-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
; GFX10W64-NEXT: .LBB1_2:
; GFX10W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX10W64-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX10W64-NEXT: s_waitcnt vmcnt(0)
-; GFX10W64-NEXT: v_readfirstlane_b32 s2, v1
+; GFX10W64-NEXT: s_mov_b32 null, 0
+; GFX10W64-NEXT: v_readfirstlane_b32 s4, v1
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10W64-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s6, v0, s[2:3]
+; GFX10W64-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s2, v0, s[4:5]
; GFX10W64-NEXT: v_mov_b32_e32 v1, 0
; GFX10W64-NEXT: global_store_dword v1, v0, s[0:1]
; GFX10W64-NEXT: s_endpgm
@@ -464,15 +466,15 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX10W32-LABEL: add_i32_uniform:
; GFX10W32: ; %bb.0: ; %entry
; GFX10W32-NEXT: s_load_dword s0, s[4:5], 0x44
-; GFX10W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W32-NEXT: s_mov_b32 s1, exec_lo
; GFX10W32-NEXT: ; implicit-def: $vgpr1
-; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX10W32-NEXT: s_bcnt1_i32_b32 s2, s1
; GFX10W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10W32-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX10W32-NEXT: s_cbranch_execz .LBB1_2
; GFX10W32-NEXT: ; %bb.1:
; GFX10W32-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX10W32-NEXT: s_mul_i32 s2, s0, s2
; GFX10W32-NEXT: v_mov_b32_e32 v1, s2
@@ -492,31 +494,32 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX11W64-LABEL: add_i32_uniform:
; GFX11W64: ; %bb.0: ; %entry
-; GFX11W64-NEXT: s_load_b32 s6, s[4:5], 0x44
-; GFX11W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX11W64-NEXT: s_load_b32 s2, s[4:5], 0x44
+; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W64-NEXT: ; implicit-def: $vgpr1
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W64-NEXT: s_cbranch_execz .LBB1_2
; GFX11W64-NEXT: ; %bb.1:
; GFX11W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11W64-NEXT: s_mul_i32 s2, s6, s2
+; GFX11W64-NEXT: s_mul_i32 s3, s2, s3
; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX11W64-NEXT: v_mov_b32_e32 v1, s3
; GFX11W64-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], 0 glc
; GFX11W64-NEXT: .LBB1_2:
; GFX11W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX11W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX11W64-NEXT: s_waitcnt vmcnt(0)
-; GFX11W64-NEXT: v_readfirstlane_b32 s2, v1
+; GFX11W64-NEXT: v_readfirstlane_b32 s4, v1
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11W64-NEXT: v_mad_u64_u32 v[1:2], null, s6, v0, s[2:3]
+; GFX11W64-NEXT: v_mad_u64_u32 v[1:2], null, s2, v0, s[4:5]
; GFX11W64-NEXT: v_mov_b32_e32 v0, 0
; GFX11W64-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX11W64-NEXT: s_endpgm
@@ -524,16 +527,16 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX11W32-LABEL: add_i32_uniform:
; GFX11W32: ; %bb.0: ; %entry
; GFX11W32-NEXT: s_load_b32 s0, s[4:5], 0x44
-; GFX11W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W32-NEXT: ; implicit-def: $vgpr1
-; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
; GFX11W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W32-NEXT: s_cbranch_execz .LBB1_2
; GFX11W32-NEXT: ; %bb.1:
; GFX11W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W32-NEXT: s_mul_i32 s2, s0, s2
; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -553,33 +556,32 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX12W64-LABEL: add_i32_uniform:
; GFX12W64: ; %bb.0: ; %entry
-; GFX12W64-NEXT: s_load_b32 s6, s[4:5], 0x44
-; GFX12W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX12W64-NEXT: s_load_b32 s2, s[4:5], 0x44
+; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W64-NEXT: ; implicit-def: $vgpr1
-; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W64-NEXT: s_cbranch_execz .LBB1_2
; GFX12W64-NEXT: ; %bb.1:
; GFX12W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX12W64-NEXT: s_wait_kmcnt 0x0
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12W64-NEXT: s_mul_i32 s2, s6, s2
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX12W64-NEXT: s_mul_i32 s3, s2, s3
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12W64-NEXT: v_mov_b32_e32 v1, s3
; GFX12W64-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
; GFX12W64-NEXT: .LBB1_2:
; GFX12W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX12W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX12W64-NEXT: s_wait_loadcnt 0x0
-; GFX12W64-NEXT: v_readfirstlane_b32 s2, v1
+; GFX12W64-NEXT: v_readfirstlane_b32 s4, v1
; GFX12W64-NEXT: s_wait_kmcnt 0x0
-; GFX12W64-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12W64-NEXT: v_mad_co_u64_u32 v[0:1], null, s6, v0, s[2:3]
+; GFX12W64-NEXT: v_mad_co_u64_u32 v[0:1], null, s2, v0, s[4:5]
; GFX12W64-NEXT: v_mov_b32_e32 v1, 0
; GFX12W64-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX12W64-NEXT: s_endpgm
@@ -587,20 +589,19 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX12W32-LABEL: add_i32_uniform:
; GFX12W32: ; %bb.0: ; %entry
; GFX12W32-NEXT: s_load_b32 s0, s[4:5], 0x44
-; GFX12W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W32-NEXT: ; implicit-def: $vgpr1
-; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
; GFX12W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W32-NEXT: s_cbranch_execz .LBB1_2
; GFX12W32-NEXT: ; %bb.1:
; GFX12W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX12W32-NEXT: s_wait_kmcnt 0x0
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W32-NEXT: s_mul_i32 s2, s0, s2
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12W32-NEXT: v_mov_b32_e32 v1, s2
; GFX12W32-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
; GFX12W32-NEXT: .LBB1_2:
@@ -617,31 +618,32 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX13W64-LABEL: add_i32_uniform:
; GFX13W64: ; %bb.0: ; %entry
-; GFX13W64-NEXT: s_load_b32 s6, s[4:5], 0x44 nv
-; GFX13W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX13W64-NEXT: s_load_b32 s2, s[4:5], 0x44 nv
+; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W64-NEXT: ; implicit-def: $vgpr1
-; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX13W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W64-NEXT: s_cbranch_execz .LBB1_2
; GFX13W64-NEXT: ; %bb.1:
; GFX13W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX13W64-NEXT: s_wait_kmcnt 0x0
-; GFX13W64-NEXT: s_mul_i32 s2, s6, s2
+; GFX13W64-NEXT: s_mul_i32 s3, s2, s3
; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX13W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX13W64-NEXT: v_mov_b32_e32 v1, s3
; GFX13W64-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
; GFX13W64-NEXT: .LBB1_2:
; GFX13W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX13W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX13W64-NEXT: s_wait_loadcnt 0x0
-; GFX13W64-NEXT: v_readfirstlane_b32 s2, v1
+; GFX13W64-NEXT: v_readfirstlane_b32 s4, v1
; GFX13W64-NEXT: s_wait_kmcnt 0x0
; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13W64-NEXT: v_mad_co_u64_u32 v[0:1], null, s6, v0, s[2:3]
+; GFX13W64-NEXT: v_mad_co_u64_u32 v[0:1], null, s2, v0, s[4:5]
; GFX13W64-NEXT: v_mov_b32_e32 v1, 0
; GFX13W64-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX13W64-NEXT: s_endpgm
@@ -649,16 +651,16 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX13W32-LABEL: add_i32_uniform:
; GFX13W32: ; %bb.0: ; %entry
; GFX13W32-NEXT: s_load_b32 s0, s[4:5], 0x44 nv
-; GFX13W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W32-NEXT: ; implicit-def: $vgpr1
-; GFX13W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
; GFX13W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W32-NEXT: s_cbranch_execz .LBB1_2
; GFX13W32-NEXT: ; %bb.1:
; GFX13W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX13W32-NEXT: s_wait_kmcnt 0x0
; GFX13W32-NEXT: s_mul_i32 s2, s0, s2
; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -1758,16 +1760,16 @@ entry:
define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace(8) %inout) {
; GFX6-LABEL: sub_i32_constant:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_mov_b64 s[2:3], exec
-; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX6-NEXT: s_mov_b64 s[0:1], exec
+; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX6-NEXT: ; implicit-def: $vgpr1
; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX6-NEXT: s_cbranch_execz .LBB5_2
; GFX6-NEXT: ; %bb.1:
; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd
-; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX6-NEXT: s_mul_i32 s2, s2, 5
; GFX6-NEXT: v_mov_b32_e32 v1, s2
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
@@ -1787,16 +1789,16 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX8-LABEL: sub_i32_constant:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB5_2
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX8-NEXT: s_mul_i32 s2, s2, 5
; GFX8-NEXT: v_mov_b32_e32 v1, s2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
@@ -1816,16 +1818,16 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX9-LABEL: sub_i32_constant:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB5_2
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_mul_i32 s2, s2, 5
; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -1844,16 +1846,16 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX10W64-LABEL: sub_i32_constant:
; GFX10W64: ; %bb.0: ; %entry
-; GFX10W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W64-NEXT: s_mov_b64 s[0:1], exec
; GFX10W64-NEXT: ; implicit-def: $vgpr1
-; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX10W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX10W64-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX10W64-NEXT: s_cbranch_execz .LBB5_2
; GFX10W64-NEXT: ; %bb.1:
; GFX10W64-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX10W64-NEXT: s_mul_i32 s2, s2, 5
; GFX10W64-NEXT: v_mov_b32_e32 v1, s2
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
@@ -1873,15 +1875,15 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX10W32-LABEL: sub_i32_constant:
; GFX10W32: ; %bb.0: ; %entry
-; GFX10W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W32-NEXT: s_mov_b32 s0, exec_lo
; GFX10W32-NEXT: ; implicit-def: $vgpr1
-; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
+; GFX10W32-NEXT: s_bcnt1_i32_b32 s1, s0
; GFX10W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10W32-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX10W32-NEXT: s_cbranch_execz .LBB5_2
; GFX10W32-NEXT: ; %bb.1:
; GFX10W32-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W32-NEXT: s_bcnt1_i32_b32 s1, s1
; GFX10W32-NEXT: s_mul_i32 s1, s1, 5
; GFX10W32-NEXT: v_mov_b32_e32 v1, s1
; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)
@@ -1901,19 +1903,20 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX11W64-LABEL: sub_i32_constant:
; GFX11W64: ; %bb.0: ; %entry
-; GFX11W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W64-NEXT: ; implicit-def: $vgpr1
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W64-NEXT: s_cbranch_execz .LBB5_2
; GFX11W64-NEXT: ; %bb.1:
; GFX11W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11W64-NEXT: s_mul_i32 s2, s2, 5
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11W64-NEXT: v_mov_b32_e32 v1, s2
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W64-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], 0 glc
@@ -1932,18 +1935,18 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX11W32-LABEL: sub_i32_constant:
; GFX11W32: ; %bb.0: ; %entry
-; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX11W32-NEXT: ; implicit-def: $vgpr1
-; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX11W32-NEXT: s_mov_b32 s0, exec_lo
; GFX11W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W32-NEXT: s_cbranch_execz .LBB5_2
; GFX11W32-NEXT: ; %bb.1:
; GFX11W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11W32-NEXT: s_mul_i32 s1, s1, 5
+; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11W32-NEXT: v_mov_b32_e32 v1, s1
; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W32-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], 0 glc
@@ -1962,20 +1965,20 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX12W64-LABEL: sub_i32_constant:
; GFX12W64: ; %bb.0: ; %entry
-; GFX12W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W64-NEXT: ; implicit-def: $vgpr1
-; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W64-NEXT: s_cbranch_execz .LBB5_2
; GFX12W64-NEXT: ; %bb.1:
; GFX12W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W64-NEXT: s_mul_i32 s2, s2, 5
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12W64-NEXT: v_mov_b32_e32 v1, s2
; GFX12W64-NEXT: s_wait_kmcnt 0x0
; GFX12W64-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -1995,19 +1998,18 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX12W32-LABEL: sub_i32_constant:
; GFX12W32: ; %bb.0: ; %entry
-; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX12W32-NEXT: ; implicit-def: $vgpr1
-; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX12W32-NEXT: s_mov_b32 s0, exec_lo
; GFX12W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W32-NEXT: s_cbranch_execz .LBB5_2
; GFX12W32-NEXT: ; %bb.1:
; GFX12W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W32-NEXT: s_mul_i32 s1, s1, 5
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12W32-NEXT: v_mov_b32_e32 v1, s1
; GFX12W32-NEXT: s_wait_kmcnt 0x0
; GFX12W32-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -2026,19 +2028,20 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX13W64-LABEL: sub_i32_constant:
; GFX13W64: ; %bb.0: ; %entry
-; GFX13W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W64-NEXT: ; implicit-def: $vgpr1
-; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX13W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W64-NEXT: s_cbranch_execz .LBB5_2
; GFX13W64-NEXT: ; %bb.1:
; GFX13W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX13W64-NEXT: s_mul_i32 s2, s2, 5
+; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX13W64-NEXT: v_mov_b32_e32 v1, s2
; GFX13W64-NEXT: s_wait_kmcnt 0x0
; GFX13W64-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -2057,18 +2060,18 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX13W32-LABEL: sub_i32_constant:
; GFX13W32: ; %bb.0: ; %entry
-; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX13W32-NEXT: ; implicit-def: $vgpr1
-; GFX13W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX13W32-NEXT: s_mov_b32 s0, exec_lo
; GFX13W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W32-NEXT: s_cbranch_execz .LBB5_2
; GFX13W32-NEXT: ; %bb.1:
; GFX13W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX13W32-NEXT: s_mul_i32 s1, s1, 5
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX13W32-NEXT: v_mov_b32_e32 v1, s1
; GFX13W32-NEXT: s_wait_kmcnt 0x0
; GFX13W32-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -2092,26 +2095,26 @@ entry:
define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(8) %inout, i32 %subitive) {
; GFX6-LABEL: sub_i32_uniform:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0x11
-; GFX6-NEXT: s_mov_b64 s[2:3], exec
-; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX6-NEXT: s_load_dword s2, s[4:5], 0x11
+; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX6-NEXT: s_mov_b64 s[0:1], exec
+; GFX6-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX6-NEXT: ; implicit-def: $vgpr1
; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX6-NEXT: s_cbranch_execz .LBB6_2
; GFX6-NEXT: ; %bb.1:
; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd
-; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mul_i32 s2, s6, s2
-; GFX6-NEXT: v_mov_b32_e32 v1, s2
+; GFX6-NEXT: s_mul_i32 s3, s2, s3
+; GFX6-NEXT: v_mov_b32_e32 v1, s3
; GFX6-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc
; GFX6-NEXT: .LBB6_2:
; GFX6-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX6-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_readfirstlane_b32 s4, v1
; GFX6-NEXT: s_mov_b32 s3, 0xf000
@@ -2122,26 +2125,26 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX8-LABEL: sub_i32_uniform:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB6_2
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mul_i32 s2, s6, s2
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
+; GFX8-NEXT: s_mul_i32 s3, s2, s3
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc
; GFX8-NEXT: .LBB6_2:
; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX8-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s2, v1
; GFX8-NEXT: v_sub_u32_e32 v2, vcc, s2, v0
@@ -2152,26 +2155,26 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX9-LABEL: sub_i32_uniform:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB6_2
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_i32 s2, s6, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: s_mul_i32 s3, s2, s3
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
; GFX9-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc
; GFX9-NEXT: .LBB6_2:
; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX9-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s2, v1
; GFX9-NEXT: v_mov_b32_e32 v2, 0
@@ -2181,27 +2184,27 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX10W64-LABEL: sub_i32_uniform:
; GFX10W64: ; %bb.0: ; %entry
-; GFX10W64-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX10W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX10W64-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W64-NEXT: s_mov_b64 s[0:1], exec
; GFX10W64-NEXT: ; implicit-def: $vgpr1
-; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX10W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX10W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX10W64-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX10W64-NEXT: s_cbranch_execz .LBB6_2
; GFX10W64-NEXT: ; %bb.1:
; GFX10W64-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10W64-NEXT: s_mul_i32 s2, s6, s2
-; GFX10W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX10W64-NEXT: s_mul_i32 s3, s2, s3
+; GFX10W64-NEXT: v_mov_b32_e32 v1, s3
; GFX10W64-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc
; GFX10W64-NEXT: .LBB6_2:
; GFX10W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX10W64-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10W64-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX10W64-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX10W64-NEXT: s_waitcnt vmcnt(0)
; GFX10W64-NEXT: v_readfirstlane_b32 s2, v1
; GFX10W64-NEXT: v_mov_b32_e32 v1, 0
@@ -2212,15 +2215,15 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX10W32-LABEL: sub_i32_uniform:
; GFX10W32: ; %bb.0: ; %entry
; GFX10W32-NEXT: s_load_dword s0, s[4:5], 0x44
-; GFX10W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W32-NEXT: s_mov_b32 s1, exec_lo
; GFX10W32-NEXT: ; implicit-def: $vgpr1
-; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX10W32-NEXT: s_bcnt1_i32_b32 s2, s1
; GFX10W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10W32-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX10W32-NEXT: s_cbranch_execz .LBB6_2
; GFX10W32-NEXT: ; %bb.1:
; GFX10W32-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX10W32-NEXT: s_mul_i32 s2, s0, s2
; GFX10W32-NEXT: v_mov_b32_e32 v1, s2
@@ -2240,28 +2243,29 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX11W64-LABEL: sub_i32_uniform:
; GFX11W64: ; %bb.0: ; %entry
-; GFX11W64-NEXT: s_load_b32 s6, s[4:5], 0x44
-; GFX11W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX11W64-NEXT: s_load_b32 s2, s[4:5], 0x44
+; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W64-NEXT: ; implicit-def: $vgpr1
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W64-NEXT: s_cbranch_execz .LBB6_2
; GFX11W64-NEXT: ; %bb.1:
; GFX11W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11W64-NEXT: s_mul_i32 s2, s6, s2
+; GFX11W64-NEXT: s_mul_i32 s3, s2, s3
; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX11W64-NEXT: v_mov_b32_e32 v1, s3
; GFX11W64-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], 0 glc
; GFX11W64-NEXT: .LBB6_2:
; GFX11W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX11W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11W64-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX11W64-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX11W64-NEXT: s_waitcnt vmcnt(0)
; GFX11W64-NEXT: v_readfirstlane_b32 s2, v1
; GFX11W64-NEXT: v_mov_b32_e32 v1, 0
@@ -2273,16 +2277,16 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX11W32-LABEL: sub_i32_uniform:
; GFX11W32: ; %bb.0: ; %entry
; GFX11W32-NEXT: s_load_b32 s0, s[4:5], 0x44
-; GFX11W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W32-NEXT: ; implicit-def: $vgpr1
-; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
; GFX11W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W32-NEXT: s_cbranch_execz .LBB6_2
; GFX11W32-NEXT: ; %bb.1:
; GFX11W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W32-NEXT: s_mul_i32 s2, s0, s2
; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -2303,29 +2307,29 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX12W64-LABEL: sub_i32_uniform:
; GFX12W64: ; %bb.0: ; %entry
-; GFX12W64-NEXT: s_load_b32 s6, s[4:5], 0x44
-; GFX12W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX12W64-NEXT: s_load_b32 s2, s[4:5], 0x44
+; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W64-NEXT: ; implicit-def: $vgpr1
-; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W64-NEXT: s_cbranch_execz .LBB6_2
; GFX12W64-NEXT: ; %bb.1:
; GFX12W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX12W64-NEXT: s_wait_kmcnt 0x0
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12W64-NEXT: s_mul_i32 s2, s6, s2
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX12W64-NEXT: s_mul_i32 s3, s2, s3
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12W64-NEXT: v_mov_b32_e32 v1, s3
; GFX12W64-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
; GFX12W64-NEXT: .LBB6_2:
; GFX12W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX12W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX12W64-NEXT: s_wait_kmcnt 0x0
-; GFX12W64-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX12W64-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX12W64-NEXT: s_wait_loadcnt 0x0
; GFX12W64-NEXT: v_readfirstlane_b32 s2, v1
; GFX12W64-NEXT: v_mov_b32_e32 v1, 0
@@ -2338,20 +2342,19 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX12W32-LABEL: sub_i32_uniform:
; GFX12W32: ; %bb.0: ; %entry
; GFX12W32-NEXT: s_load_b32 s0, s[4:5], 0x44
-; GFX12W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W32-NEXT: ; implicit-def: $vgpr1
-; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
; GFX12W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W32-NEXT: s_cbranch_execz .LBB6_2
; GFX12W32-NEXT: ; %bb.1:
; GFX12W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX12W32-NEXT: s_wait_kmcnt 0x0
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W32-NEXT: s_mul_i32 s2, s0, s2
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12W32-NEXT: v_mov_b32_e32 v1, s2
; GFX12W32-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
; GFX12W32-NEXT: .LBB6_2:
@@ -2370,28 +2373,29 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX13W64-LABEL: sub_i32_uniform:
; GFX13W64: ; %bb.0: ; %entry
-; GFX13W64-NEXT: s_load_b32 s6, s[4:5], 0x44 nv
-; GFX13W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX13W64-NEXT: s_load_b32 s2, s[4:5], 0x44 nv
+; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W64-NEXT: ; implicit-def: $vgpr1
-; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX13W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W64-NEXT: s_cbranch_execz .LBB6_2
; GFX13W64-NEXT: ; %bb.1:
; GFX13W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX13W64-NEXT: s_wait_kmcnt 0x0
-; GFX13W64-NEXT: s_mul_i32 s2, s6, s2
+; GFX13W64-NEXT: s_mul_i32 s3, s2, s3
; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX13W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX13W64-NEXT: v_mov_b32_e32 v1, s3
; GFX13W64-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
; GFX13W64-NEXT: .LBB6_2:
; GFX13W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX13W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX13W64-NEXT: s_wait_kmcnt 0x0
-; GFX13W64-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX13W64-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX13W64-NEXT: s_wait_loadcnt 0x0
; GFX13W64-NEXT: v_readfirstlane_b32 s2, v1
; GFX13W64-NEXT: v_mov_b32_e32 v1, 0
@@ -2403,16 +2407,16 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX13W32-LABEL: sub_i32_uniform:
; GFX13W32: ; %bb.0: ; %entry
; GFX13W32-NEXT: s_load_b32 s0, s[4:5], 0x44 nv
-; GFX13W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W32-NEXT: ; implicit-def: $vgpr1
-; GFX13W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
; GFX13W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W32-NEXT: s_cbranch_execz .LBB6_2
; GFX13W32-NEXT: ; %bb.1:
; GFX13W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX13W32-NEXT: s_wait_kmcnt 0x0
; GFX13W32-NEXT: s_mul_i32 s2, s0, s2
; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index 8c21d42c05a2c..1796272dd4b9f 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -38,22 +38,22 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX7LESS-LABEL: add_i32_constant:
; GFX7LESS: ; %bb.0: ; %entry
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX7LESS-NEXT: s_mov_b64 s[4:5], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX7LESS-NEXT: ; implicit-def: $vgpr1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB0_2
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_mov_b32 s8, s2
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
-; GFX7LESS-NEXT: s_mul_i32 s2, s2, 5
+; GFX7LESS-NEXT: s_mul_i32 s6, s6, 5
; GFX7LESS-NEXT: s_mov_b32 s11, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s10, -1
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_mov_b32 s8, s2
; GFX7LESS-NEXT: s_mov_b32 s9, s3
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s2
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s6
; GFX7LESS-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS-NEXT: buffer_wbinvl1
@@ -70,22 +70,22 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX8-LABEL: add_i32_constant:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b64 s[6:7], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[4:5], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX8-NEXT: s_cbranch_execz .LBB0_2
; GFX8-NEXT: ; %bb.1:
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mov_b32 s8, s2
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
-; GFX8-NEXT: s_mul_i32 s2, s2, 5
+; GFX8-NEXT: s_mul_i32 s6, s6, 5
; GFX8-NEXT: s_mov_b32 s11, 0xf000
; GFX8-NEXT: s_mov_b32 s10, -1
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_mov_b32 s8, s2
; GFX8-NEXT: s_mov_b32 s9, s3
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s6
; GFX8-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
@@ -102,22 +102,22 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX9-LABEL: add_i32_constant:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b64 s[6:7], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_cbranch_execz .LBB0_2
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mov_b32 s8, s2
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
-; GFX9-NEXT: s_mul_i32 s2, s2, 5
+; GFX9-NEXT: s_mul_i32 s6, s6, 5
; GFX9-NEXT: s_mov_b32 s11, 0xf000
; GFX9-NEXT: s_mov_b32 s10, -1
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 s8, s2
; GFX9-NEXT: s_mov_b32 s9, s3
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: v_mov_b32_e32 v1, s6
; GFX9-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -134,19 +134,19 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1064-LABEL: add_i32_constant:
; GFX1064: ; %bb.0: ; %entry
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1064-NEXT: s_mov_b64 s[6:7], exec
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], exec
; GFX1064-NEXT: ; implicit-def: $vgpr1
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB0_2
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1064-NEXT: s_mov_b32 s11, 0x31016000
; GFX1064-NEXT: s_mul_i32 s6, s6, 5
-; GFX1064-NEXT: s_mov_b32 s10, -1
+; GFX1064-NEXT: s_mov_b32 s11, 0x31016000
; GFX1064-NEXT: v_mov_b32_e32 v1, s6
+; GFX1064-NEXT: s_mov_b32 s10, -1
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_mov_b32 s8, s2
; GFX1064-NEXT: s_mov_b32 s9, s3
@@ -168,18 +168,18 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1032-LABEL: add_i32_constant:
; GFX1032: ; %bb.0: ; %entry
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1032-NEXT: s_mov_b32 s6, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s4, exec_lo
; GFX1032-NEXT: ; implicit-def: $vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s4
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB0_2
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s6
-; GFX1032-NEXT: s_mov_b32 s11, 0x31016000
; GFX1032-NEXT: s_mul_i32 s5, s5, 5
-; GFX1032-NEXT: s_mov_b32 s10, -1
+; GFX1032-NEXT: s_mov_b32 s11, 0x31016000
; GFX1032-NEXT: v_mov_b32_e32 v1, s5
+; GFX1032-NEXT: s_mov_b32 s10, -1
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_mov_b32 s8, s2
; GFX1032-NEXT: s_mov_b32 s9, s3
@@ -201,20 +201,21 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1164-LABEL: add_i32_constant:
; GFX1164: ; %bb.0: ; %entry
; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1164-NEXT: s_mov_b64 s[6:7], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[4:5], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1164-NEXT: ; implicit-def: $vgpr1
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1164-NEXT: s_mov_b64 s[4:5], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-NEXT: s_cbranch_execz .LBB0_2
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1164-NEXT: s_mov_b32 s11, 0x31016000
; GFX1164-NEXT: s_mul_i32 s6, s6, 5
-; GFX1164-NEXT: s_mov_b32 s10, -1
+; GFX1164-NEXT: s_mov_b32 s11, 0x31016000
; GFX1164-NEXT: v_mov_b32_e32 v1, s6
+; GFX1164-NEXT: s_mov_b32 s10, -1
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_mov_b32 s8, s2
; GFX1164-NEXT: s_mov_b32 s9, s3
@@ -235,19 +236,19 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1132-LABEL: add_i32_constant:
; GFX1132: ; %bb.0: ; %entry
; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1132-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s4, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1132-NEXT: ; implicit-def: $vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_bcnt1_i32_b32 s5, s4
+; GFX1132-NEXT: s_mov_b32 s4, exec_lo
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB0_2
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s5, s6
-; GFX1132-NEXT: s_mov_b32 s11, 0x31016000
; GFX1132-NEXT: s_mul_i32 s5, s5, 5
-; GFX1132-NEXT: s_mov_b32 s10, -1
+; GFX1132-NEXT: s_mov_b32 s11, 0x31016000
; GFX1132-NEXT: v_mov_b32_e32 v1, s5
+; GFX1132-NEXT: s_mov_b32 s10, -1
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_mov_b32 s8, s2
; GFX1132-NEXT: s_mov_b32 s9, s3
@@ -268,22 +269,21 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1264-LABEL: add_i32_constant:
; GFX1264: ; %bb.0: ; %entry
; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1264-NEXT: s_mov_b64 s[6:7], exec
+; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1264-NEXT: s_mov_b64 s[4:5], exec
-; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1264-NEXT: ; implicit-def: $vgpr1
-; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX1264-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1264-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1264-NEXT: s_mov_b64 s[4:5], exec
+; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1264-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1264-NEXT: s_cbranch_execz .LBB0_2
; GFX1264-NEXT: ; %bb.1:
-; GFX1264-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1264-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1264-NEXT: s_mul_i32 s6, s6, 5
-; GFX1264-NEXT: s_mov_b32 s10, -1
-; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1264-NEXT: s_mov_b32 s11, 0x31016000
; GFX1264-NEXT: v_mov_b32_e32 v1, s6
+; GFX1264-NEXT: s_mov_b32 s10, -1
; GFX1264-NEXT: s_wait_kmcnt 0x0
; GFX1264-NEXT: s_mov_b32 s8, s2
; GFX1264-NEXT: s_mov_b32 s9, s3
@@ -303,19 +303,19 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1232-LABEL: add_i32_constant:
; GFX1232: ; %bb.0: ; %entry
; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1232-NEXT: s_mov_b32 s6, exec_lo
+; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1232-NEXT: s_mov_b32 s4, exec_lo
-; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1232-NEXT: ; implicit-def: $vgpr1
-; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1232-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: s_bcnt1_i32_b32 s5, s4
+; GFX1232-NEXT: s_mov_b32 s4, exec_lo
; GFX1232-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1232-NEXT: s_cbranch_execz .LBB0_2
; GFX1232-NEXT: ; %bb.1:
-; GFX1232-NEXT: s_bcnt1_i32_b32 s5, s6
-; GFX1232-NEXT: s_mov_b32 s11, 0x31016000
; GFX1232-NEXT: s_mul_i32 s5, s5, 5
-; GFX1232-NEXT: s_mov_b32 s10, -1
+; GFX1232-NEXT: s_mov_b32 s11, 0x31016000
; GFX1232-NEXT: v_mov_b32_e32 v1, s5
+; GFX1232-NEXT: s_mov_b32 s10, -1
; GFX1232-NEXT: s_wait_kmcnt 0x0
; GFX1232-NEXT: s_mov_b32 s8, s2
; GFX1232-NEXT: s_mov_b32 s9, s3
@@ -335,20 +335,21 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1364-LABEL: add_i32_constant:
; GFX1364: ; %bb.0: ; %entry
; GFX1364-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1364-NEXT: s_mov_b64 s[6:7], exec
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_mov_b64 s[4:5], exec
-; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1364-NEXT: ; implicit-def: $vgpr1
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1364-NEXT: s_mov_b64 s[4:5], exec
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1364-NEXT: s_cbranch_execz .LBB0_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1364-NEXT: s_mov_b32 s11, 0x31016000
; GFX1364-NEXT: s_mul_i32 s6, s6, 5
-; GFX1364-NEXT: s_mov_b32 s10, -1
+; GFX1364-NEXT: s_mov_b32 s11, 0x31016000
; GFX1364-NEXT: v_mov_b32_e32 v1, s6
+; GFX1364-NEXT: s_mov_b32 s10, -1
; GFX1364-NEXT: s_wait_kmcnt 0x0
; GFX1364-NEXT: s_mov_b32 s8, s2
; GFX1364-NEXT: s_mov_b32 s9, s3
@@ -371,19 +372,19 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1332-LABEL: add_i32_constant:
; GFX1332: ; %bb.0: ; %entry
; GFX1332-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1332-NEXT: s_mov_b32 s6, exec_lo
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1332-NEXT: s_mov_b32 s4, exec_lo
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1332-NEXT: ; implicit-def: $vgpr1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: s_bcnt1_i32_b32 s5, s4
+; GFX1332-NEXT: s_mov_b32 s4, exec_lo
; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1332-NEXT: s_cbranch_execz .LBB0_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: s_bcnt1_i32_b32 s5, s6
-; GFX1332-NEXT: s_mov_b32 s11, 0x31016000
; GFX1332-NEXT: s_mul_i32 s5, s5, 5
-; GFX1332-NEXT: s_mov_b32 s10, -1
+; GFX1332-NEXT: s_mov_b32 s11, 0x31016000
; GFX1332-NEXT: v_mov_b32_e32 v1, s5
+; GFX1332-NEXT: s_mov_b32 s10, -1
; GFX1332-NEXT: s_wait_kmcnt 0x0
; GFX1332-NEXT: s_mov_b32 s8, s2
; GFX1332-NEXT: s_mov_b32 s9, s3
@@ -412,30 +413,30 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX7LESS-LABEL: add_i32_uniform:
; GFX7LESS: ; %bb.0: ; %entry
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7LESS-NEXT: s_load_dword s8, s[4:5], 0xd
-; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
+; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0xd
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX7LESS-NEXT: s_mov_b64 s[4:5], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX7LESS-NEXT: ; implicit-def: $vgpr1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB1_2
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_mov_b32 s12, s2
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
-; GFX7LESS-NEXT: s_mul_i32 s2, s8, s2
-; GFX7LESS-NEXT: s_mov_b32 s15, 0xf000
-; GFX7LESS-NEXT: s_mov_b32 s14, -1
-; GFX7LESS-NEXT: s_mov_b32 s13, s3
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s2
-; GFX7LESS-NEXT: buffer_atomic_add v1, off, s[12:15], 0 glc
+; GFX7LESS-NEXT: s_mul_i32 s7, s6, s7
+; GFX7LESS-NEXT: s_mov_b32 s11, 0xf000
+; GFX7LESS-NEXT: s_mov_b32 s10, -1
+; GFX7LESS-NEXT: s_mov_b32 s8, s2
+; GFX7LESS-NEXT: s_mov_b32 s9, s3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s7
+; GFX7LESS-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS-NEXT: buffer_wbinvl1
; GFX7LESS-NEXT: .LBB1_2:
; GFX7LESS-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mul_lo_u32 v0, s8, v0
+; GFX7LESS-NEXT: v_mul_lo_u32 v0, s6, v0
; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v1
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
@@ -446,30 +447,30 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX8-LABEL: add_i32_uniform:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_load_dword s8, s[4:5], 0x34
-; GFX8-NEXT: s_mov_b64 s[6:7], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX8-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[4:5], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX8-NEXT: s_cbranch_execz .LBB1_2
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mov_b32 s12, s2
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
-; GFX8-NEXT: s_mul_i32 s2, s8, s2
-; GFX8-NEXT: s_mov_b32 s15, 0xf000
-; GFX8-NEXT: s_mov_b32 s14, -1
-; GFX8-NEXT: s_mov_b32 s13, s3
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
-; GFX8-NEXT: buffer_atomic_add v1, off, s[12:15], 0 glc
+; GFX8-NEXT: s_mul_i32 s7, s6, s7
+; GFX8-NEXT: s_mov_b32 s11, 0xf000
+; GFX8-NEXT: s_mov_b32 s10, -1
+; GFX8-NEXT: s_mov_b32 s8, s2
+; GFX8-NEXT: s_mov_b32 s9, s3
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: .LBB1_2:
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mul_lo_u32 v0, s8, v0
+; GFX8-NEXT: v_mul_lo_u32 v0, s6, v0
; GFX8-NEXT: v_readfirstlane_b32 s4, v1
; GFX8-NEXT: s_mov_b32 s3, 0xf000
; GFX8-NEXT: s_mov_b32 s2, -1
@@ -480,30 +481,30 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX9-LABEL: add_i32_uniform:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s8, s[4:5], 0x34
-; GFX9-NEXT: s_mov_b64 s[6:7], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX9-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_cbranch_execz .LBB1_2
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mov_b32 s12, s2
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
-; GFX9-NEXT: s_mul_i32 s2, s8, s2
-; GFX9-NEXT: s_mov_b32 s15, 0xf000
-; GFX9-NEXT: s_mov_b32 s14, -1
-; GFX9-NEXT: s_mov_b32 s13, s3
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
-; GFX9-NEXT: buffer_atomic_add v1, off, s[12:15], 0 glc
+; GFX9-NEXT: s_mul_i32 s7, s6, s7
+; GFX9-NEXT: s_mov_b32 s11, 0xf000
+; GFX9-NEXT: s_mov_b32 s10, -1
+; GFX9-NEXT: s_mov_b32 s8, s2
+; GFX9-NEXT: s_mov_b32 s9, s3
+; GFX9-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: .LBB1_2:
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mul_lo_u32 v0, s8, v0
+; GFX9-NEXT: v_mul_lo_u32 v0, s6, v0
; GFX9-NEXT: v_readfirstlane_b32 s4, v1
; GFX9-NEXT: s_mov_b32 s3, 0xf000
; GFX9-NEXT: s_mov_b32 s2, -1
@@ -515,24 +516,24 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1064: ; %bb.0: ; %entry
; GFX1064-NEXT: s_clause 0x1
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1064-NEXT: s_load_dword s8, s[4:5], 0x34
-; GFX1064-NEXT: s_mov_b64 s[6:7], exec
+; GFX1064-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], exec
; GFX1064-NEXT: ; implicit-def: $vgpr1
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB1_2
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1064-NEXT: s_mov_b32 s15, 0x31016000
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_mul_i32 s6, s8, s6
-; GFX1064-NEXT: s_mov_b32 s14, -1
-; GFX1064-NEXT: v_mov_b32_e32 v1, s6
-; GFX1064-NEXT: s_mov_b32 s12, s2
-; GFX1064-NEXT: s_mov_b32 s13, s3
-; GFX1064-NEXT: buffer_atomic_add v1, off, s[12:15], 0 glc
+; GFX1064-NEXT: s_mul_i32 s7, s6, s7
+; GFX1064-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1064-NEXT: v_mov_b32_e32 v1, s7
+; GFX1064-NEXT: s_mov_b32 s10, -1
+; GFX1064-NEXT: s_mov_b32 s8, s2
+; GFX1064-NEXT: s_mov_b32 s9, s3
+; GFX1064-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: buffer_gl1_inv
; GFX1064-NEXT: buffer_gl0_inv
@@ -541,7 +542,7 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_readfirstlane_b32 s2, v1
-; GFX1064-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s8, v0, s[2:3]
+; GFX1064-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s6, v0, s[2:3]
; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
; GFX1064-NEXT: s_mov_b32 s2, -1
; GFX1064-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -552,19 +553,19 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1032-NEXT: s_clause 0x1
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032-NEXT: s_load_dword s6, s[4:5], 0x34
-; GFX1032-NEXT: s_mov_b32 s7, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s4, exec_lo
; GFX1032-NEXT: ; implicit-def: $vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s7, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s4
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB1_2
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s7
-; GFX1032-NEXT: s_mov_b32 s11, 0x31016000
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_mul_i32 s5, s6, s5
-; GFX1032-NEXT: s_mov_b32 s10, -1
+; GFX1032-NEXT: s_mov_b32 s11, 0x31016000
; GFX1032-NEXT: v_mov_b32_e32 v1, s5
+; GFX1032-NEXT: s_mov_b32 s10, -1
; GFX1032-NEXT: s_mov_b32 s8, s2
; GFX1032-NEXT: s_mov_b32 s9, s3
; GFX1032-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
@@ -586,25 +587,26 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1164: ; %bb.0: ; %entry
; GFX1164-NEXT: s_clause 0x1
; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1164-NEXT: s_load_b32 s8, s[4:5], 0x34
-; GFX1164-NEXT: s_mov_b64 s[6:7], exec
+; GFX1164-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[4:5], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1164-NEXT: ; implicit-def: $vgpr1
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
+; GFX1164-NEXT: s_mov_b64 s[4:5], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-NEXT: s_cbranch_execz .LBB1_2
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1164-NEXT: s_mov_b32 s15, 0x31016000
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_mul_i32 s6, s8, s6
-; GFX1164-NEXT: s_mov_b32 s14, -1
-; GFX1164-NEXT: v_mov_b32_e32 v1, s6
-; GFX1164-NEXT: s_mov_b32 s12, s2
-; GFX1164-NEXT: s_mov_b32 s13, s3
-; GFX1164-NEXT: buffer_atomic_add_u32 v1, off, s[12:15], 0 glc
+; GFX1164-NEXT: s_mul_i32 s7, s6, s7
+; GFX1164-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1164-NEXT: v_mov_b32_e32 v1, s7
+; GFX1164-NEXT: s_mov_b32 s10, -1
+; GFX1164-NEXT: s_mov_b32 s8, s2
+; GFX1164-NEXT: s_mov_b32 s9, s3
+; GFX1164-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], 0 glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: buffer_gl1_inv
; GFX1164-NEXT: buffer_gl0_inv
@@ -613,7 +615,7 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: v_readfirstlane_b32 s2, v1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_mad_u64_u32 v[1:2], null, s8, v0, s[2:3]
+; GFX1164-NEXT: v_mad_u64_u32 v[1:2], null, s6, v0, s[2:3]
; GFX1164-NEXT: s_mov_b32 s3, 0x31016000
; GFX1164-NEXT: s_mov_b32 s2, -1
; GFX1164-NEXT: buffer_store_b32 v1, off, s[0:3], 0
@@ -624,20 +626,20 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132-NEXT: s_load_b32 s4, s[4:5], 0x34
-; GFX1132-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s5, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1132-NEXT: ; implicit-def: $vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_bcnt1_i32_b32 s6, s5
+; GFX1132-NEXT: s_mov_b32 s5, exec_lo
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB1_2
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s6, s6
-; GFX1132-NEXT: s_mov_b32 s11, 0x31016000
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_mul_i32 s6, s4, s6
-; GFX1132-NEXT: s_mov_b32 s10, -1
+; GFX1132-NEXT: s_mov_b32 s11, 0x31016000
; GFX1132-NEXT: v_mov_b32_e32 v1, s6
+; GFX1132-NEXT: s_mov_b32 s10, -1
; GFX1132-NEXT: s_mov_b32 s8, s2
; GFX1132-NEXT: s_mov_b32 s9, s3
; GFX1132-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], 0 glc
@@ -659,27 +661,26 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1264: ; %bb.0: ; %entry
; GFX1264-NEXT: s_clause 0x1
; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1264-NEXT: s_load_b32 s8, s[4:5], 0x34
-; GFX1264-NEXT: s_mov_b64 s[6:7], exec
+; GFX1264-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1264-NEXT: s_mov_b64 s[4:5], exec
-; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1264-NEXT: ; implicit-def: $vgpr1
-; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX1264-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1264-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
+; GFX1264-NEXT: s_mov_b64 s[4:5], exec
+; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1264-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1264-NEXT: s_cbranch_execz .LBB1_2
; GFX1264-NEXT: ; %bb.1:
-; GFX1264-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1264-NEXT: s_mov_b32 s15, 0x31016000
; GFX1264-NEXT: s_wait_kmcnt 0x0
-; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-NEXT: s_mul_i32 s6, s8, s6
-; GFX1264-NEXT: s_mov_b32 s14, -1
-; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-NEXT: v_mov_b32_e32 v1, s6
-; GFX1264-NEXT: s_mov_b32 s12, s2
-; GFX1264-NEXT: s_mov_b32 s13, s3
-; GFX1264-NEXT: buffer_atomic_add_u32 v1, off, s[12:15], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1264-NEXT: s_mul_i32 s7, s6, s7
+; GFX1264-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1264-NEXT: v_mov_b32_e32 v1, s7
+; GFX1264-NEXT: s_mov_b32 s10, -1
+; GFX1264-NEXT: s_mov_b32 s8, s2
+; GFX1264-NEXT: s_mov_b32 s9, s3
+; GFX1264-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1264-NEXT: s_wait_loadcnt 0x0
; GFX1264-NEXT: global_inv scope:SCOPE_DEV
; GFX1264-NEXT: .LBB1_2:
@@ -687,7 +688,7 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1264-NEXT: s_wait_kmcnt 0x0
; GFX1264-NEXT: v_readfirstlane_b32 s2, v1
; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1264-NEXT: v_mad_co_u64_u32 v[0:1], null, s8, v0, s[2:3]
+; GFX1264-NEXT: v_mad_co_u64_u32 v[0:1], null, s6, v0, s[2:3]
; GFX1264-NEXT: s_mov_b32 s3, 0x31016000
; GFX1264-NEXT: s_mov_b32 s2, -1
; GFX1264-NEXT: buffer_store_b32 v0, off, s[0:3], null
@@ -698,22 +699,20 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1232-NEXT: s_clause 0x1
; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1232-NEXT: s_load_b32 s4, s[4:5], 0x34
-; GFX1232-NEXT: s_mov_b32 s6, exec_lo
+; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1232-NEXT: s_mov_b32 s5, exec_lo
-; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1232-NEXT: ; implicit-def: $vgpr1
-; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1232-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: s_bcnt1_i32_b32 s6, s5
+; GFX1232-NEXT: s_mov_b32 s5, exec_lo
; GFX1232-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1232-NEXT: s_cbranch_execz .LBB1_2
; GFX1232-NEXT: ; %bb.1:
-; GFX1232-NEXT: s_bcnt1_i32_b32 s6, s6
-; GFX1232-NEXT: s_mov_b32 s11, 0x31016000
; GFX1232-NEXT: s_wait_kmcnt 0x0
-; GFX1232-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1232-NEXT: s_mul_i32 s6, s4, s6
-; GFX1232-NEXT: s_mov_b32 s10, -1
-; GFX1232-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1232-NEXT: s_mov_b32 s11, 0x31016000
; GFX1232-NEXT: v_mov_b32_e32 v1, s6
+; GFX1232-NEXT: s_mov_b32 s10, -1
; GFX1232-NEXT: s_mov_b32 s8, s2
; GFX1232-NEXT: s_mov_b32 s9, s3
; GFX1232-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -734,27 +733,28 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1364: ; %bb.0: ; %entry
; GFX1364-NEXT: s_clause 0x1
; GFX1364-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1364-NEXT: s_load_b32 s8, s[4:5], 0x34 nv
-; GFX1364-NEXT: s_mov_b64 s[6:7], exec
+; GFX1364-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_mov_b64 s[4:5], exec
-; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1364-NEXT: ; implicit-def: $vgpr1
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
+; GFX1364-NEXT: s_mov_b64 s[4:5], exec
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1364-NEXT: s_cbranch_execz .LBB1_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1364-NEXT: s_mov_b32 s15, 0x31016000
; GFX1364-NEXT: s_wait_kmcnt 0x0
-; GFX1364-NEXT: s_mul_i32 s6, s8, s6
-; GFX1364-NEXT: s_mov_b32 s14, -1
-; GFX1364-NEXT: v_mov_b32_e32 v1, s6
-; GFX1364-NEXT: s_mov_b32 s12, s2
-; GFX1364-NEXT: s_mov_b32 s13, s3
+; GFX1364-NEXT: s_mul_i32 s7, s6, s7
+; GFX1364-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1364-NEXT: v_mov_b32_e32 v1, s7
+; GFX1364-NEXT: s_mov_b32 s10, -1
+; GFX1364-NEXT: s_mov_b32 s8, s2
+; GFX1364-NEXT: s_mov_b32 s9, s3
; GFX1364-NEXT: global_wb scope:SCOPE_DEV
; GFX1364-NEXT: s_wait_storecnt 0x0
-; GFX1364-NEXT: buffer_atomic_add_u32 v1, off, s[12:15], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1364-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1364-NEXT: s_wait_loadcnt 0x0
; GFX1364-NEXT: global_inv scope:SCOPE_DEV
; GFX1364-NEXT: s_wait_loadcnt 0x0
@@ -763,7 +763,7 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1364-NEXT: s_wait_kmcnt 0x0
; GFX1364-NEXT: v_readfirstlane_b32 s2, v1
; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1364-NEXT: v_mad_co_u64_u32 v[0:1], null, s8, v0, s[2:3]
+; GFX1364-NEXT: v_mad_co_u64_u32 v[0:1], null, s6, v0, s[2:3]
; GFX1364-NEXT: s_mov_b32 s3, 0x31016000
; GFX1364-NEXT: s_mov_b32 s2, -1
; GFX1364-NEXT: buffer_store_b32 v0, off, s[0:3], null
@@ -774,20 +774,20 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1332-NEXT: s_clause 0x1
; GFX1332-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1332-NEXT: s_load_b32 s4, s[4:5], 0x34 nv
-; GFX1332-NEXT: s_mov_b32 s6, exec_lo
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1332-NEXT: s_mov_b32 s5, exec_lo
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1332-NEXT: ; implicit-def: $vgpr1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: s_bcnt1_i32_b32 s6, s5
+; GFX1332-NEXT: s_mov_b32 s5, exec_lo
; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1332-NEXT: s_cbranch_execz .LBB1_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: s_bcnt1_i32_b32 s6, s6
-; GFX1332-NEXT: s_mov_b32 s11, 0x31016000
; GFX1332-NEXT: s_wait_kmcnt 0x0
; GFX1332-NEXT: s_mul_i32 s6, s4, s6
-; GFX1332-NEXT: s_mov_b32 s10, -1
+; GFX1332-NEXT: s_mov_b32 s11, 0x31016000
; GFX1332-NEXT: v_mov_b32_e32 v1, s6
+; GFX1332-NEXT: s_mov_b32 s10, -1
; GFX1332-NEXT: s_mov_b32 s8, s2
; GFX1332-NEXT: s_mov_b32 s9, s3
; GFX1332-NEXT: global_wb scope:SCOPE_DEV
@@ -1850,24 +1850,28 @@ entry:
define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace(1) %inout) {
; GFX7LESS-LABEL: add_i64_constant:
; GFX7LESS: ; %bb.0: ; %entry
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[0:1]
+; GFX7LESS-NEXT: v_mul_hi_u32 v0, 5, s6
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, s7, v0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v1
+; GFX7LESS-NEXT: v_readfirstlane_b32 s7, v0
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX7LESS-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB3_2
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_mov_b32 s8, s2
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
-; GFX7LESS-NEXT: s_mul_i32 s2, s2, 5
+; GFX7LESS-NEXT: s_mul_i32 s12, s6, 5
+; GFX7LESS-NEXT: s_mul_i32 s6, s6, 0
+; GFX7LESS-NEXT: s_add_u32 s13, s7, s6
; GFX7LESS-NEXT: s_mov_b32 s11, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s10, -1
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_mov_b32 s8, s2
; GFX7LESS-NEXT: s_mov_b32 s9, s3
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s12
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s13
; GFX7LESS-NEXT: buffer_atomic_add_x2 v[0:1], off, s[8:11], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS-NEXT: buffer_wbinvl1
@@ -1889,24 +1893,28 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX8-LABEL: add_i64_constant:
; GFX8: ; %bb.0: ; %entry
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[0:1]
+; GFX8-NEXT: v_mul_hi_u32 v0, 5, s6
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b64 s[6:7], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v1
+; GFX8-NEXT: v_readfirstlane_b32 s7, v0
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX8-NEXT: s_cbranch_execz .LBB3_2
; GFX8-NEXT: ; %bb.1:
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mov_b32 s8, s2
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
-; GFX8-NEXT: s_mul_i32 s2, s2, 5
+; GFX8-NEXT: s_mul_i32 s12, s6, 5
+; GFX8-NEXT: s_mul_i32 s6, s6, 0
+; GFX8-NEXT: s_add_u32 s13, s7, s6
; GFX8-NEXT: s_mov_b32 s11, 0xf000
; GFX8-NEXT: s_mov_b32 s10, -1
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_mov_b32 s8, s2
; GFX8-NEXT: s_mov_b32 s9, s3
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_mov_b32_e32 v1, 0
+; GFX8-NEXT: v_mov_b32_e32 v0, s12
+; GFX8-NEXT: v_mov_b32_e32 v1, s13
; GFX8-NEXT: buffer_atomic_add_x2 v[0:1], off, s[8:11], 0 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
@@ -1928,23 +1936,26 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX9-LABEL: add_i64_constant:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b64 s[6:7], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_cbranch_execz .LBB3_2
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mov_b32 s8, s2
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
-; GFX9-NEXT: s_mul_i32 s2, s2, 5
+; GFX9-NEXT: s_mul_i32 s12, s6, 5
+; GFX9-NEXT: s_mul_hi_u32 s7, 5, s6
+; GFX9-NEXT: s_mul_i32 s6, s6, 0
+; GFX9-NEXT: s_add_u32 s13, s7, s6
; GFX9-NEXT: s_mov_b32 s11, 0xf000
; GFX9-NEXT: s_mov_b32 s10, -1
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 s8, s2
; GFX9-NEXT: s_mov_b32 s9, s3
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: v_mov_b32_e32 v0, s12
+; GFX9-NEXT: v_mov_b32_e32 v1, s13
; GFX9-NEXT: buffer_atomic_add_x2 v[0:1], off, s[8:11], 0 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -1965,19 +1976,22 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1064-LABEL: add_i64_constant:
; GFX1064: ; %bb.0: ; %entry
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1064-NEXT: s_mov_b64 s[6:7], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], exec
+; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1064-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB3_2
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1064-NEXT: v_mov_b32_e32 v1, 0
+; GFX1064-NEXT: s_mul_hi_u32 s7, 5, s6
+; GFX1064-NEXT: s_mul_i32 s8, s6, 0
; GFX1064-NEXT: s_mul_i32 s6, s6, 5
-; GFX1064-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1064-NEXT: s_add_u32 s7, s7, s8
; GFX1064-NEXT: v_mov_b32_e32 v0, s6
+; GFX1064-NEXT: v_mov_b32_e32 v1, s7
+; GFX1064-NEXT: s_mov_b32 s11, 0x31016000
; GFX1064-NEXT: s_mov_b32 s10, -1
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_mov_b32 s8, s2
@@ -2001,18 +2015,21 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1032-LABEL: add_i64_constant:
; GFX1032: ; %bb.0: ; %entry
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1032-NEXT: s_mov_b32 s6, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s4, exec_lo
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s4
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB3_2
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s6
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0
-; GFX1032-NEXT: s_mul_i32 s5, s5, 5
+; GFX1032-NEXT: s_mul_hi_u32 s7, 5, s5
+; GFX1032-NEXT: s_mul_i32 s8, s5, 0
+; GFX1032-NEXT: s_mul_i32 s6, s5, 5
+; GFX1032-NEXT: s_add_u32 s7, s7, s8
+; GFX1032-NEXT: v_mov_b32_e32 v0, s6
+; GFX1032-NEXT: v_mov_b32_e32 v1, s7
; GFX1032-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1032-NEXT: v_mov_b32_e32 v0, s5
; GFX1032-NEXT: s_mov_b32 s10, -1
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_mov_b32 s8, s2
@@ -2036,20 +2053,24 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1164-LABEL: add_i64_constant:
; GFX1164: ; %bb.0: ; %entry
; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1164-NEXT: s_mov_b64 s[6:7], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[4:5], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1164-NEXT: s_mov_b64 s[4:5], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1164-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB3_2
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0
+; GFX1164-NEXT: s_mul_hi_u32 s7, 5, s6
+; GFX1164-NEXT: s_mul_i32 s8, s6, 0
; GFX1164-NEXT: s_mul_i32 s6, s6, 5
-; GFX1164-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1164-NEXT: s_add_u32 s7, s7, s8
; GFX1164-NEXT: v_mov_b32_e32 v0, s6
+; GFX1164-NEXT: v_mov_b32_e32 v1, s7
+; GFX1164-NEXT: s_mov_b32 s11, 0x31016000
; GFX1164-NEXT: s_mov_b32 s10, -1
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_mov_b32 s8, s2
@@ -2073,19 +2094,23 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1132-LABEL: add_i64_constant:
; GFX1132: ; %bb.0: ; %entry
; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1132-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s4, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_bcnt1_i32_b32 s5, s4
+; GFX1132-NEXT: s_mov_b32 s4, exec_lo
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB3_2
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s5, s6
+; GFX1132-NEXT: s_mul_hi_u32 s7, 5, s5
+; GFX1132-NEXT: s_mul_i32 s8, s5, 0
+; GFX1132-NEXT: s_mul_i32 s6, s5, 5
+; GFX1132-NEXT: s_add_u32 s7, s7, s8
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
; GFX1132-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1132-NEXT: s_mul_i32 s5, s5, 5
; GFX1132-NEXT: s_mov_b32 s10, -1
-; GFX1132-NEXT: v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v1, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_mov_b32 s8, s2
; GFX1132-NEXT: s_mov_b32 s9, s3
@@ -2108,22 +2133,24 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1264-LABEL: add_i64_constant:
; GFX1264: ; %bb.0: ; %entry
; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1264-NEXT: s_mov_b64 s[6:7], exec
+; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1264-NEXT: s_mov_b64 s[4:5], exec
+; GFX1264-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1264-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX1264-NEXT: s_mov_b64 s[4:5], exec
-; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1264-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1264-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1264-NEXT: s_cbranch_execz .LBB3_2
; GFX1264-NEXT: ; %bb.1:
-; GFX1264-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1264-NEXT: v_mov_b32_e32 v1, 0
-; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1264-NEXT: s_mul_hi_u32 s7, 5, s6
+; GFX1264-NEXT: s_mul_i32 s8, s6, 0
; GFX1264-NEXT: s_mul_i32 s6, s6, 5
-; GFX1264-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1264-NEXT: s_add_co_u32 s7, s7, s8
; GFX1264-NEXT: v_mov_b32_e32 v0, s6
+; GFX1264-NEXT: v_mov_b32_e32 v1, s7
+; GFX1264-NEXT: s_mov_b32 s11, 0x31016000
; GFX1264-NEXT: s_mov_b32 s10, -1
; GFX1264-NEXT: s_wait_kmcnt 0x0
; GFX1264-NEXT: s_mov_b32 s8, s2
@@ -2146,19 +2173,23 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1232-LABEL: add_i64_constant:
; GFX1232: ; %bb.0: ; %entry
; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1232-NEXT: s_mov_b32 s6, exec_lo
+; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1232-NEXT: s_mov_b32 s4, exec_lo
-; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
; GFX1232-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1232-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: s_bcnt1_i32_b32 s5, s4
+; GFX1232-NEXT: s_mov_b32 s4, exec_lo
; GFX1232-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1232-NEXT: s_cbranch_execz .LBB3_2
; GFX1232-NEXT: ; %bb.1:
-; GFX1232-NEXT: s_bcnt1_i32_b32 s5, s6
+; GFX1232-NEXT: s_mul_hi_u32 s7, 5, s5
+; GFX1232-NEXT: s_mul_i32 s8, s5, 0
+; GFX1232-NEXT: s_mul_i32 s6, s5, 5
+; GFX1232-NEXT: s_add_co_u32 s7, s7, s8
+; GFX1232-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1232-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
; GFX1232-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1232-NEXT: s_mul_i32 s5, s5, 5
; GFX1232-NEXT: s_mov_b32 s10, -1
-; GFX1232-NEXT: v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v1, 0
; GFX1232-NEXT: s_wait_kmcnt 0x0
; GFX1232-NEXT: s_mov_b32 s8, s2
; GFX1232-NEXT: s_mov_b32 s9, s3
@@ -2180,20 +2211,24 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1364-LABEL: add_i64_constant:
; GFX1364: ; %bb.0: ; %entry
; GFX1364-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1364-NEXT: s_mov_b64 s[6:7], exec
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_mov_b64 s[4:5], exec
-; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1364-NEXT: s_mov_b64 s[4:5], exec
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1364-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1364-NEXT: s_cbranch_execz .LBB3_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1364-NEXT: v_mov_b32_e32 v1, 0
+; GFX1364-NEXT: s_mul_hi_u32 s7, 5, s6
+; GFX1364-NEXT: s_mul_i32 s8, s6, 0
; GFX1364-NEXT: s_mul_i32 s6, s6, 5
-; GFX1364-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1364-NEXT: s_add_co_u32 s7, s7, s8
; GFX1364-NEXT: v_mov_b32_e32 v0, s6
+; GFX1364-NEXT: v_mov_b32_e32 v1, s7
+; GFX1364-NEXT: s_mov_b32 s11, 0x31016000
; GFX1364-NEXT: s_mov_b32 s10, -1
; GFX1364-NEXT: s_wait_kmcnt 0x0
; GFX1364-NEXT: s_mov_b32 s8, s2
@@ -2219,19 +2254,23 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1332-LABEL: add_i64_constant:
; GFX1332: ; %bb.0: ; %entry
; GFX1332-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1332-NEXT: s_mov_b32 s6, exec_lo
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1332-NEXT: s_mov_b32 s4, exec_lo
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
; GFX1332-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: s_bcnt1_i32_b32 s5, s4
+; GFX1332-NEXT: s_mov_b32 s4, exec_lo
; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1332-NEXT: s_cbranch_execz .LBB3_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: s_bcnt1_i32_b32 s5, s6
+; GFX1332-NEXT: s_mul_hi_u32 s7, 5, s5
+; GFX1332-NEXT: s_mul_i32 s8, s5, 0
+; GFX1332-NEXT: s_mul_i32 s6, s5, 5
+; GFX1332-NEXT: s_add_co_u32 s7, s7, s8
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1332-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
; GFX1332-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1332-NEXT: s_mul_i32 s5, s5, 5
; GFX1332-NEXT: s_mov_b32 s10, -1
-; GFX1332-NEXT: v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v1, 0
; GFX1332-NEXT: s_wait_kmcnt 0x0
; GFX1332-NEXT: s_mov_b32 s8, s2
; GFX1332-NEXT: s_mov_b32 s9, s3
@@ -2261,40 +2300,42 @@ entry:
define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, ptr addrspace(1) %inout, i64 %additive) {
; GFX7LESS-LABEL: add_i64_uniform:
; GFX7LESS: ; %bb.0: ; %entry
+; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s8, s[0:1]
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s8
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7LESS-NEXT: s_mov_b64 s[8:9], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s8, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, s9, v0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mul_hi_u32 v0, s6, v0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v1
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: v_readfirstlane_b32 s9, v0
; GFX7LESS-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB4_2
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_mov_b32 s12, s2
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[8:9]
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2
-; GFX7LESS-NEXT: v_mul_hi_u32 v0, s4, v0
-; GFX7LESS-NEXT: s_mov_b32 s13, s3
-; GFX7LESS-NEXT: s_mul_i32 s3, s5, s2
-; GFX7LESS-NEXT: s_mul_i32 s2, s4, s2
-; GFX7LESS-NEXT: s_mov_b32 s15, 0xf000
-; GFX7LESS-NEXT: s_mov_b32 s14, -1
-; GFX7LESS-NEXT: v_add_i32_e32 v1, vcc, s3, v0
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2
-; GFX7LESS-NEXT: buffer_atomic_add_x2 v[0:1], off, s[12:15], 0 glc
+; GFX7LESS-NEXT: s_mul_i32 s12, s6, s8
+; GFX7LESS-NEXT: s_mul_i32 s8, s7, s8
+; GFX7LESS-NEXT: s_add_u32 s13, s9, s8
+; GFX7LESS-NEXT: s_mov_b32 s11, 0xf000
+; GFX7LESS-NEXT: s_mov_b32 s10, -1
+; GFX7LESS-NEXT: s_mov_b32 s8, s2
+; GFX7LESS-NEXT: s_mov_b32 s9, s3
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s12
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s13
+; GFX7LESS-NEXT: buffer_atomic_add_x2 v[0:1], off, s[8:11], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS-NEXT: buffer_wbinvl1
; GFX7LESS-NEXT: .LBB4_2:
-; GFX7LESS-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v1
-; GFX7LESS-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
-; GFX7LESS-NEXT: v_mul_lo_u32 v1, s5, v2
-; GFX7LESS-NEXT: v_mul_hi_u32 v3, s4, v2
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v1
+; GFX7LESS-NEXT: s_waitcnt expcnt(0)
+; GFX7LESS-NEXT: v_mul_lo_u32 v1, s7, v2
+; GFX7LESS-NEXT: v_mul_hi_u32 v3, s6, v2
; GFX7LESS-NEXT: v_readfirstlane_b32 s5, v0
-; GFX7LESS-NEXT: v_mul_lo_u32 v0, s4, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, s6
+; GFX7LESS-NEXT: v_mul_lo_u32 v0, s6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s4
; GFX7LESS-NEXT: v_add_i32_e32 v1, vcc, v3, v1
; GFX7LESS-NEXT: v_add_i32_e32 v0, vcc, s5, v0
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
@@ -2305,38 +2346,41 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX8-LABEL: add_i64_uniform:
; GFX8: ; %bb.0: ; %entry
+; GFX8-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s8, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
-; GFX8-NEXT: s_mov_b64 s[8:9], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s8, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, s9, v0
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mul_hi_u32 v0, s6, v0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX8-NEXT: v_readfirstlane_b32 s9, v0
; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX8-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX8-NEXT: s_cbranch_execz .LBB4_2
; GFX8-NEXT: ; %bb.1:
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mov_b32 s12, s2
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[8:9]
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[8:9], s4, v0, 0
-; GFX8-NEXT: s_mul_i32 s2, s5, s2
-; GFX8-NEXT: s_mov_b32 s15, 0xf000
-; GFX8-NEXT: s_mov_b32 s14, -1
-; GFX8-NEXT: s_mov_b32 s13, s3
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, s2, v1
-; GFX8-NEXT: buffer_atomic_add_x2 v[0:1], off, s[12:15], 0 glc
+; GFX8-NEXT: s_mul_i32 s12, s6, s8
+; GFX8-NEXT: s_mul_i32 s8, s7, s8
+; GFX8-NEXT: s_add_u32 s13, s9, s8
+; GFX8-NEXT: s_mov_b32 s11, 0xf000
+; GFX8-NEXT: s_mov_b32 s10, -1
+; GFX8-NEXT: s_mov_b32 s8, s2
+; GFX8-NEXT: s_mov_b32 s9, s3
+; GFX8-NEXT: v_mov_b32_e32 v0, s12
+; GFX8-NEXT: v_mov_b32_e32 v1, s13
+; GFX8-NEXT: buffer_atomic_add_x2 v[0:1], off, s[8:11], 0 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: .LBB4_2:
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_readfirstlane_b32 s2, v1
; GFX8-NEXT: v_readfirstlane_b32 s3, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s3
; GFX8-NEXT: v_mov_b32_e32 v1, s2
-; GFX8-NEXT: v_mul_lo_u32 v3, s5, v2
-; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s4, v2, v[0:1]
+; GFX8-NEXT: v_mul_lo_u32 v3, s7, v2
+; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s6, v2, v[0:1]
; GFX8-NEXT: s_mov_b32 s3, 0xf000
; GFX8-NEXT: s_mov_b32 s2, -1
; GFX8-NEXT: v_add_u32_e32 v1, vcc, v3, v1
@@ -2348,27 +2392,27 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX9-NEXT: s_mov_b64 s[8:9], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s8, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, s9, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s8, s[4:5]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_cbranch_execz .LBB4_2
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mov_b32 s12, s2
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[8:9]
-; GFX9-NEXT: s_mov_b32 s13, s3
-; GFX9-NEXT: s_mul_i32 s3, s7, s2
-; GFX9-NEXT: s_mul_hi_u32 s8, s6, s2
-; GFX9-NEXT: s_add_i32 s8, s8, s3
-; GFX9-NEXT: s_mul_i32 s2, s6, s2
-; GFX9-NEXT: s_mov_b32 s15, 0xf000
-; GFX9-NEXT: s_mov_b32 s14, -1
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s8
-; GFX9-NEXT: buffer_atomic_add_x2 v[0:1], off, s[12:15], 0 glc
+; GFX9-NEXT: s_mul_i32 s12, s6, s8
+; GFX9-NEXT: s_mul_hi_u32 s9, s6, s8
+; GFX9-NEXT: s_mul_i32 s8, s7, s8
+; GFX9-NEXT: s_add_u32 s13, s9, s8
+; GFX9-NEXT: s_mov_b32 s11, 0xf000
+; GFX9-NEXT: s_mov_b32 s10, -1
+; GFX9-NEXT: s_mov_b32 s8, s2
+; GFX9-NEXT: s_mov_b32 s9, s3
+; GFX9-NEXT: v_mov_b32_e32 v0, s12
+; GFX9-NEXT: v_mov_b32_e32 v1, s13
+; GFX9-NEXT: buffer_atomic_add_x2 v[0:1], off, s[8:11], 0 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: .LBB4_2:
@@ -2391,23 +2435,23 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1064-NEXT: s_clause 0x1
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX1064-NEXT: s_mov_b64 s[8:9], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s8, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, s9, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], exec
+; GFX1064-NEXT: s_bcnt1_i32_b64 s8, s[4:5]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1064-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB4_2
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s8, s[8:9]
-; GFX1064-NEXT: s_mov_b32 s11, 0x31016000
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_mul_i32 s9, s7, s8
-; GFX1064-NEXT: s_mul_hi_u32 s10, s6, s8
+; GFX1064-NEXT: s_mul_hi_u32 s9, s6, s8
+; GFX1064-NEXT: s_mul_i32 s10, s7, s8
; GFX1064-NEXT: s_mul_i32 s8, s6, s8
-; GFX1064-NEXT: s_add_i32 s10, s10, s9
+; GFX1064-NEXT: s_add_u32 s9, s9, s10
; GFX1064-NEXT: v_mov_b32_e32 v0, s8
-; GFX1064-NEXT: v_mov_b32_e32 v1, s10
+; GFX1064-NEXT: v_mov_b32_e32 v1, s9
+; GFX1064-NEXT: s_mov_b32 s11, 0x31016000
; GFX1064-NEXT: s_mov_b32 s10, -1
; GFX1064-NEXT: s_mov_b32 s8, s2
; GFX1064-NEXT: s_mov_b32 s9, s3
@@ -2433,22 +2477,22 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1032-NEXT: s_clause 0x1
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX1032-NEXT: s_mov_b32 s8, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s4, exec_lo
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, s8, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s4
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB4_2
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s8
-; GFX1032-NEXT: s_mov_b32 s11, 0x31016000
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_mul_i32 s8, s7, s5
; GFX1032-NEXT: s_mul_hi_u32 s9, s6, s5
-; GFX1032-NEXT: s_mul_i32 s5, s6, s5
-; GFX1032-NEXT: s_add_i32 s9, s9, s8
-; GFX1032-NEXT: v_mov_b32_e32 v0, s5
+; GFX1032-NEXT: s_mul_i32 s10, s7, s5
+; GFX1032-NEXT: s_mul_i32 s8, s6, s5
+; GFX1032-NEXT: s_add_u32 s9, s9, s10
+; GFX1032-NEXT: v_mov_b32_e32 v0, s8
; GFX1032-NEXT: v_mov_b32_e32 v1, s9
+; GFX1032-NEXT: s_mov_b32 s11, 0x31016000
; GFX1032-NEXT: s_mov_b32 s10, -1
; GFX1032-NEXT: s_mov_b32 s8, s2
; GFX1032-NEXT: s_mov_b32 s9, s3
@@ -2474,24 +2518,25 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1164-NEXT: s_clause 0x1
; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
-; GFX1164-NEXT: s_mov_b64 s[8:9], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[6:7], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s8, 0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, s9, v0
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_bcnt1_i32_b64 s8, s[6:7]
+; GFX1164-NEXT: s_mov_b64 s[6:7], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1164-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB4_2
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s8, s[8:9]
-; GFX1164-NEXT: s_mov_b32 s11, 0x31016000
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_mul_i32 s9, s5, s8
-; GFX1164-NEXT: s_mul_hi_u32 s10, s4, s8
+; GFX1164-NEXT: s_mul_hi_u32 s9, s4, s8
+; GFX1164-NEXT: s_mul_i32 s10, s5, s8
; GFX1164-NEXT: s_mul_i32 s8, s4, s8
-; GFX1164-NEXT: s_add_i32 s10, s10, s9
+; GFX1164-NEXT: s_add_u32 s9, s9, s10
; GFX1164-NEXT: v_mov_b32_e32 v0, s8
-; GFX1164-NEXT: v_mov_b32_e32 v1, s10
+; GFX1164-NEXT: v_mov_b32_e32 v1, s9
+; GFX1164-NEXT: s_mov_b32 s11, 0x31016000
; GFX1164-NEXT: s_mov_b32 s10, -1
; GFX1164-NEXT: s_mov_b32 s8, s2
; GFX1164-NEXT: s_mov_b32 s9, s3
@@ -2518,23 +2563,23 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
-; GFX1132-NEXT: s_mov_b32 s7, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s6, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, s7, 0
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_bcnt1_i32_b32 s7, s6
+; GFX1132-NEXT: s_mov_b32 s6, exec_lo
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB4_2
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s7, s7
-; GFX1132-NEXT: s_mov_b32 s11, 0x31016000
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_mul_i32 s8, s5, s7
; GFX1132-NEXT: s_mul_hi_u32 s9, s4, s7
-; GFX1132-NEXT: s_mul_i32 s7, s4, s7
-; GFX1132-NEXT: s_add_i32 s9, s9, s8
+; GFX1132-NEXT: s_mul_i32 s10, s5, s7
+; GFX1132-NEXT: s_mul_i32 s8, s4, s7
+; GFX1132-NEXT: s_add_u32 s9, s9, s10
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, s7 :: v_dual_mov_b32 v1, s9
+; GFX1132-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
+; GFX1132-NEXT: s_mov_b32 s11, 0x31016000
; GFX1132-NEXT: s_mov_b32 s10, -1
; GFX1132-NEXT: s_mov_b32 s8, s2
; GFX1132-NEXT: s_mov_b32 s9, s3
@@ -2561,23 +2606,25 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1264-NEXT: s_clause 0x1
; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1264-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
-; GFX1264-NEXT: s_mov_b64 s[8:9], exec
-; GFX1264-NEXT: s_mov_b32 s11, 0
-; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, s8, 0
+; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1264-NEXT: s_mov_b64 s[6:7], exec
-; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v2, s9, v0
+; GFX1264-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1264-NEXT: s_bcnt1_i32_b64 s8, s[6:7]
+; GFX1264-NEXT: s_mov_b64 s[6:7], exec
+; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1264-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1264-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1264-NEXT: s_cbranch_execz .LBB4_2
; GFX1264-NEXT: ; %bb.1:
-; GFX1264-NEXT: s_bcnt1_i32_b64 s10, s[8:9]
; GFX1264-NEXT: s_wait_kmcnt 0x0
-; GFX1264-NEXT: s_mul_u64 s[8:9], s[4:5], s[10:11]
-; GFX1264-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1264-NEXT: s_mul_hi_u32 s9, s4, s8
+; GFX1264-NEXT: s_mul_i32 s10, s5, s8
+; GFX1264-NEXT: s_mul_i32 s8, s4, s8
+; GFX1264-NEXT: s_add_co_u32 s9, s9, s10
; GFX1264-NEXT: v_mov_b32_e32 v0, s8
; GFX1264-NEXT: v_mov_b32_e32 v1, s9
+; GFX1264-NEXT: s_mov_b32 s11, 0x31016000
; GFX1264-NEXT: s_mov_b32 s10, -1
; GFX1264-NEXT: s_mov_b32 s8, s2
; GFX1264-NEXT: s_mov_b32 s9, s3
@@ -2602,30 +2649,31 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1232-NEXT: s_clause 0x1
; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1232-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1232-NEXT: s_mov_b32 s6, exec_lo
-; GFX1232-NEXT: s_mov_b32 s7, 0
-; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
-; GFX1232-NEXT: s_mov_b32 s8, exec_lo
; GFX1232-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1232-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: s_bcnt1_i32_b32 s7, s6
+; GFX1232-NEXT: s_mov_b32 s6, exec_lo
; GFX1232-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1232-NEXT: s_cbranch_execz .LBB4_2
; GFX1232-NEXT: ; %bb.1:
-; GFX1232-NEXT: s_bcnt1_i32_b32 s6, s6
-; GFX1232-NEXT: s_mov_b32 s15, 0x31016000
; GFX1232-NEXT: s_wait_kmcnt 0x0
-; GFX1232-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232-NEXT: s_mul_u64 s[6:7], s[4:5], s[6:7]
-; GFX1232-NEXT: s_mov_b32 s14, -1
-; GFX1232-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
-; GFX1232-NEXT: s_mov_b32 s12, s2
-; GFX1232-NEXT: s_mov_b32 s13, s3
-; GFX1232-NEXT: buffer_atomic_add_u64 v[0:1], off, s[12:15], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1232-NEXT: s_mul_hi_u32 s9, s4, s7
+; GFX1232-NEXT: s_mul_i32 s10, s5, s7
+; GFX1232-NEXT: s_mul_i32 s8, s4, s7
+; GFX1232-NEXT: s_add_co_u32 s9, s9, s10
+; GFX1232-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1232-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
+; GFX1232-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1232-NEXT: s_mov_b32 s10, -1
+; GFX1232-NEXT: s_mov_b32 s8, s2
+; GFX1232-NEXT: s_mov_b32 s9, s3
+; GFX1232-NEXT: buffer_atomic_add_u64 v[0:1], off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1232-NEXT: s_wait_loadcnt 0x0
; GFX1232-NEXT: global_inv scope:SCOPE_DEV
; GFX1232-NEXT: .LBB4_2:
-; GFX1232-NEXT: s_or_b32 exec_lo, exec_lo, s8
+; GFX1232-NEXT: s_or_b32 exec_lo, exec_lo, s6
; GFX1232-NEXT: s_wait_kmcnt 0x0
; GFX1232-NEXT: v_readfirstlane_b32 s3, v1
; GFX1232-NEXT: v_readfirstlane_b32 s2, v0
@@ -2642,22 +2690,25 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1364-NEXT: s_clause 0x1
; GFX1364-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1364-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
-; GFX1364-NEXT: s_mov_b64 s[8:9], exec
-; GFX1364-NEXT: s_mov_b32 s11, 0
-; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, s8, 0
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_mov_b64 s[6:7], exec
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v2, s9, v0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_bcnt1_i32_b64 s8, s[6:7]
+; GFX1364-NEXT: s_mov_b64 s[6:7], exec
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1364-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1364-NEXT: s_cbranch_execz .LBB4_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: s_bcnt1_i32_b64 s10, s[8:9]
; GFX1364-NEXT: s_wait_kmcnt 0x0
-; GFX1364-NEXT: s_mul_u64 s[8:9], s[4:5], s[10:11]
-; GFX1364-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1364-NEXT: s_mul_hi_u32 s9, s4, s8
+; GFX1364-NEXT: s_mul_i32 s10, s5, s8
+; GFX1364-NEXT: s_mul_i32 s8, s4, s8
+; GFX1364-NEXT: s_add_co_u32 s9, s9, s10
; GFX1364-NEXT: v_mov_b32_e32 v0, s8
; GFX1364-NEXT: v_mov_b32_e32 v1, s9
+; GFX1364-NEXT: s_mov_b32 s11, 0x31016000
; GFX1364-NEXT: s_mov_b32 s10, -1
; GFX1364-NEXT: s_mov_b32 s8, s2
; GFX1364-NEXT: s_mov_b32 s9, s3
@@ -2685,31 +2736,34 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1332-NEXT: s_clause 0x1
; GFX1332-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1332-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1332-NEXT: s_mov_b32 s6, exec_lo
-; GFX1332-NEXT: s_mov_b32 s7, 0
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
-; GFX1332-NEXT: s_mov_b32 s8, exec_lo
; GFX1332-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: s_bcnt1_i32_b32 s7, s6
+; GFX1332-NEXT: s_mov_b32 s6, exec_lo
; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1332-NEXT: s_cbranch_execz .LBB4_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: s_bcnt1_i32_b32 s6, s6
-; GFX1332-NEXT: s_mov_b32 s15, 0x31016000
; GFX1332-NEXT: s_wait_kmcnt 0x0
-; GFX1332-NEXT: s_mul_u64 s[6:7], s[4:5], s[6:7]
-; GFX1332-NEXT: s_mov_b32 s14, -1
-; GFX1332-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
-; GFX1332-NEXT: s_mov_b32 s12, s2
-; GFX1332-NEXT: s_mov_b32 s13, s3
+; GFX1332-NEXT: s_mul_hi_u32 s9, s4, s7
+; GFX1332-NEXT: s_mul_i32 s10, s5, s7
+; GFX1332-NEXT: s_mul_i32 s8, s4, s7
+; GFX1332-NEXT: s_add_co_u32 s9, s9, s10
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1332-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
+; GFX1332-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1332-NEXT: s_mov_b32 s10, -1
+; GFX1332-NEXT: s_mov_b32 s8, s2
+; GFX1332-NEXT: s_mov_b32 s9, s3
; GFX1332-NEXT: global_wb scope:SCOPE_DEV
; GFX1332-NEXT: s_wait_storecnt 0x0
-; GFX1332-NEXT: buffer_atomic_add_u64 v[0:1], off, s[12:15], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1332-NEXT: buffer_atomic_add_u64 v[0:1], off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1332-NEXT: s_wait_loadcnt 0x0
; GFX1332-NEXT: global_inv scope:SCOPE_DEV
; GFX1332-NEXT: s_wait_loadcnt 0x0
; GFX1332-NEXT: .LBB4_2:
-; GFX1332-NEXT: s_or_b32 exec_lo, exec_lo, s8
+; GFX1332-NEXT: s_or_b32 exec_lo, exec_lo, s6
; GFX1332-NEXT: s_wait_kmcnt 0x0
; GFX1332-NEXT: v_readfirstlane_b32 s3, v1
; GFX1332-NEXT: v_readfirstlane_b32 s2, v0
@@ -4183,23 +4237,23 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX7LESS-LABEL: sub_i32_constant:
; GFX7LESS: ; %bb.0: ; %entry
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, s7, v0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v0
+; GFX7LESS-NEXT: s_mov_b64 s[4:5], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX7LESS-NEXT: ; implicit-def: $vgpr0
; GFX7LESS-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB6_4
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dword s4, s[2:3], 0x0
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s12, s[6:7]
+; GFX7LESS-NEXT: s_load_dword s5, s[2:3], 0x0
+; GFX7LESS-NEXT: s_mul_i32 s12, s4, 5
; GFX7LESS-NEXT: s_mov_b64 s[10:11], 0
; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
-; GFX7LESS-NEXT: s_mul_i32 s12, s12, 5
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s4
; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s5
; GFX7LESS-NEXT: s_mov_b32 s4, s2
; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: .LBB6_2: ; %atomicrmw.start
@@ -4233,23 +4287,23 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX8-LABEL: sub_i32_constant:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b64 s[6:7], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[4:5], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX8-NEXT: ; implicit-def: $vgpr0
; GFX8-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX8-NEXT: s_cbranch_execz .LBB6_4
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s4, s[2:3], 0x0
-; GFX8-NEXT: s_bcnt1_i32_b64 s12, s[6:7]
+; GFX8-NEXT: s_load_dword s5, s[2:3], 0x0
+; GFX8-NEXT: s_mul_i32 s12, s4, 5
; GFX8-NEXT: s_mov_b64 s[10:11], 0
; GFX8-NEXT: s_mov_b32 s7, 0xf000
-; GFX8-NEXT: s_mul_i32 s12, s12, 5
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
; GFX8-NEXT: s_mov_b32 s6, -1
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v0, s5
; GFX8-NEXT: s_mov_b32 s4, s2
; GFX8-NEXT: s_mov_b32 s5, s3
; GFX8-NEXT: .LBB6_2: ; %atomicrmw.start
@@ -4281,23 +4335,23 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX9-LABEL: sub_i32_constant:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b64 s[6:7], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX9-NEXT: ; implicit-def: $vgpr0
; GFX9-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX9-NEXT: s_cbranch_execz .LBB6_4
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dword s4, s[2:3], 0x0
-; GFX9-NEXT: s_bcnt1_i32_b64 s12, s[6:7]
+; GFX9-NEXT: s_load_dword s5, s[2:3], 0x0
+; GFX9-NEXT: s_mul_i32 s12, s4, 5
; GFX9-NEXT: s_mov_b64 s[10:11], 0
; GFX9-NEXT: s_mov_b32 s7, 0xf000
-; GFX9-NEXT: s_mul_i32 s12, s12, 5
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v0, s5
; GFX9-NEXT: s_mov_b32 s4, s2
; GFX9-NEXT: s_mov_b32 s5, s3
; GFX9-NEXT: .LBB6_2: ; %atomicrmw.start
@@ -4329,25 +4383,25 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1064-LABEL: sub_i32_constant:
; GFX1064: ; %bb.0: ; %entry
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1064-NEXT: s_mov_b64 s[6:7], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], exec
+; GFX1064-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1064-NEXT: ; implicit-def: $vgpr0
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX1064-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB6_4
; GFX1064-NEXT: ; %bb.1:
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_load_dword s4, s[2:3], 0x0
-; GFX1064-NEXT: s_bcnt1_i32_b64 s12, s[6:7]
+; GFX1064-NEXT: s_load_dword s5, s[2:3], 0x0
+; GFX1064-NEXT: s_mul_i32 s12, s4, 5
; GFX1064-NEXT: s_mov_b64 s[10:11], 0
-; GFX1064-NEXT: s_mul_i32 s12, s12, 5
; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
; GFX1064-NEXT: s_mov_b32 s6, -1
-; GFX1064-NEXT: s_mov_b32 s5, s3
-; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v0, s4
; GFX1064-NEXT: s_mov_b32 s4, s2
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: v_mov_b32_e32 v0, s5
+; GFX1064-NEXT: s_mov_b32 s5, s3
; GFX1064-NEXT: .LBB6_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: v_mov_b32_e32 v4, v0
@@ -4378,24 +4432,24 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1032-LABEL: sub_i32_constant:
; GFX1032: ; %bb.0: ; %entry
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1032-NEXT: s_mov_b32 s6, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s4, exec_lo
; GFX1032-NEXT: s_mov_b32 s9, 0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s4, s4
; GFX1032-NEXT: ; implicit-def: $vgpr0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
; GFX1032-NEXT: s_and_saveexec_b32 s8, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB6_4
; GFX1032-NEXT: ; %bb.1:
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dword s4, s[2:3], 0x0
-; GFX1032-NEXT: s_bcnt1_i32_b32 s10, s6
+; GFX1032-NEXT: s_load_dword s5, s[2:3], 0x0
+; GFX1032-NEXT: s_mul_i32 s10, s4, 5
; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1032-NEXT: s_mul_i32 s10, s10, 5
; GFX1032-NEXT: s_mov_b32 s6, -1
-; GFX1032-NEXT: s_mov_b32 s5, s3
-; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mov_b32_e32 v0, s4
; GFX1032-NEXT: s_mov_b32 s4, s2
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: v_mov_b32_e32 v0, s5
+; GFX1032-NEXT: s_mov_b32 s5, s3
; GFX1032-NEXT: .LBB6_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: v_mov_b32_e32 v4, v0
@@ -4426,26 +4480,26 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1164-LABEL: sub_i32_constant:
; GFX1164: ; %bb.0: ; %entry
; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1164-NEXT: s_mov_b64 s[6:7], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: s_mov_b64 s[4:5], exec
; GFX1164-NEXT: s_mov_b64 s[8:9], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1164-NEXT: ; implicit-def: $vgpr0
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB6_4
; GFX1164-NEXT: ; %bb.1:
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_load_b32 s4, s[2:3], 0x0
-; GFX1164-NEXT: s_bcnt1_i32_b64 s12, s[6:7]
+; GFX1164-NEXT: s_load_b32 s5, s[2:3], 0x0
+; GFX1164-NEXT: s_mul_i32 s12, s4, 5
; GFX1164-NEXT: s_mov_b64 s[10:11], 0
-; GFX1164-NEXT: s_mul_i32 s12, s12, 5
; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
; GFX1164-NEXT: s_mov_b32 s6, -1
-; GFX1164-NEXT: s_mov_b32 s5, s3
-; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mov_b32_e32 v0, s4
; GFX1164-NEXT: s_mov_b32 s4, s2
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: v_mov_b32_e32 v0, s5
+; GFX1164-NEXT: s_mov_b32 s5, s3
; GFX1164-NEXT: .LBB6_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -4480,25 +4534,24 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1132-LABEL: sub_i32_constant:
; GFX1132: ; %bb.0: ; %entry
; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1132-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s4, exec_lo
; GFX1132-NEXT: s_mov_b32 s9, 0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s4, s4
; GFX1132-NEXT: s_mov_b32 s8, exec_lo
; GFX1132-NEXT: ; implicit-def: $vgpr0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB6_4
; GFX1132-NEXT: ; %bb.1:
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b32 s4, s[2:3], 0x0
-; GFX1132-NEXT: s_bcnt1_i32_b32 s10, s6
+; GFX1132-NEXT: s_load_b32 s5, s[2:3], 0x0
+; GFX1132-NEXT: s_mul_i32 s10, s4, 5
; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1132-NEXT: s_mul_i32 s10, s10, 5
; GFX1132-NEXT: s_mov_b32 s6, -1
-; GFX1132-NEXT: s_mov_b32 s5, s3
-; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: v_mov_b32_e32 v0, s4
; GFX1132-NEXT: s_mov_b32 s4, s2
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: v_mov_b32_e32 v0, s5
+; GFX1132-NEXT: s_mov_b32 s5, s3
; GFX1132-NEXT: .LBB6_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -4532,22 +4585,21 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1264-LABEL: sub_i32_constant:
; GFX1264: ; %bb.0: ; %entry
; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1264-NEXT: s_mov_b64 s[6:7], exec
+; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1264-NEXT: s_mov_b64 s[4:5], exec
-; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1264-NEXT: ; implicit-def: $vgpr1
-; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX1264-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1264-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1264-NEXT: s_mov_b64 s[4:5], exec
+; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1264-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1264-NEXT: s_cbranch_execz .LBB6_2
; GFX1264-NEXT: ; %bb.1:
-; GFX1264-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1264-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1264-NEXT: s_mul_i32 s6, s6, 5
-; GFX1264-NEXT: s_mov_b32 s10, -1
-; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1264-NEXT: s_mov_b32 s11, 0x31016000
; GFX1264-NEXT: v_mov_b32_e32 v1, s6
+; GFX1264-NEXT: s_mov_b32 s10, -1
; GFX1264-NEXT: s_wait_kmcnt 0x0
; GFX1264-NEXT: s_mov_b32 s8, s2
; GFX1264-NEXT: s_mov_b32 s9, s3
@@ -4569,19 +4621,19 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1232-LABEL: sub_i32_constant:
; GFX1232: ; %bb.0: ; %entry
; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1232-NEXT: s_mov_b32 s6, exec_lo
+; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1232-NEXT: s_mov_b32 s4, exec_lo
-; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1232-NEXT: ; implicit-def: $vgpr1
-; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1232-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: s_bcnt1_i32_b32 s5, s4
+; GFX1232-NEXT: s_mov_b32 s4, exec_lo
; GFX1232-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1232-NEXT: s_cbranch_execz .LBB6_2
; GFX1232-NEXT: ; %bb.1:
-; GFX1232-NEXT: s_bcnt1_i32_b32 s5, s6
-; GFX1232-NEXT: s_mov_b32 s11, 0x31016000
; GFX1232-NEXT: s_mul_i32 s5, s5, 5
-; GFX1232-NEXT: s_mov_b32 s10, -1
+; GFX1232-NEXT: s_mov_b32 s11, 0x31016000
; GFX1232-NEXT: v_mov_b32_e32 v1, s5
+; GFX1232-NEXT: s_mov_b32 s10, -1
; GFX1232-NEXT: s_wait_kmcnt 0x0
; GFX1232-NEXT: s_mov_b32 s8, s2
; GFX1232-NEXT: s_mov_b32 s9, s3
@@ -4603,20 +4655,21 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1364-LABEL: sub_i32_constant:
; GFX1364: ; %bb.0: ; %entry
; GFX1364-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1364-NEXT: s_mov_b64 s[6:7], exec
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_mov_b64 s[4:5], exec
-; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1364-NEXT: ; implicit-def: $vgpr1
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1364-NEXT: s_mov_b64 s[4:5], exec
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1364-NEXT: s_cbranch_execz .LBB6_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1364-NEXT: s_mov_b32 s11, 0x31016000
; GFX1364-NEXT: s_mul_i32 s6, s6, 5
-; GFX1364-NEXT: s_mov_b32 s10, -1
+; GFX1364-NEXT: s_mov_b32 s11, 0x31016000
; GFX1364-NEXT: v_mov_b32_e32 v1, s6
+; GFX1364-NEXT: s_mov_b32 s10, -1
; GFX1364-NEXT: s_wait_kmcnt 0x0
; GFX1364-NEXT: s_mov_b32 s8, s2
; GFX1364-NEXT: s_mov_b32 s9, s3
@@ -4641,19 +4694,19 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1332-LABEL: sub_i32_constant:
; GFX1332: ; %bb.0: ; %entry
; GFX1332-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1332-NEXT: s_mov_b32 s6, exec_lo
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1332-NEXT: s_mov_b32 s4, exec_lo
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1332-NEXT: ; implicit-def: $vgpr1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: s_bcnt1_i32_b32 s5, s4
+; GFX1332-NEXT: s_mov_b32 s4, exec_lo
; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1332-NEXT: s_cbranch_execz .LBB6_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: s_bcnt1_i32_b32 s5, s6
-; GFX1332-NEXT: s_mov_b32 s11, 0x31016000
; GFX1332-NEXT: s_mul_i32 s5, s5, 5
-; GFX1332-NEXT: s_mov_b32 s10, -1
+; GFX1332-NEXT: s_mov_b32 s11, 0x31016000
; GFX1332-NEXT: v_mov_b32_e32 v1, s5
+; GFX1332-NEXT: s_mov_b32 s10, -1
; GFX1332-NEXT: s_wait_kmcnt 0x0
; GFX1332-NEXT: s_mov_b32 s8, s2
; GFX1332-NEXT: s_mov_b32 s9, s3
@@ -4685,23 +4738,23 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX7LESS: ; %bb.0: ; %entry
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX7LESS-NEXT: s_load_dword s12, s[4:5], 0xd
-; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, s7, v0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v0
+; GFX7LESS-NEXT: s_mov_b64 s[4:5], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX7LESS-NEXT: ; implicit-def: $vgpr0
; GFX7LESS-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB7_4
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dword s4, s[2:3], 0x0
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s5, s[6:7]
+; GFX7LESS-NEXT: s_load_dword s5, s[2:3], 0x0
+; GFX7LESS-NEXT: s_mul_i32 s13, s12, s4
; GFX7LESS-NEXT: s_mov_b64 s[10:11], 0
; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
-; GFX7LESS-NEXT: s_mul_i32 s13, s12, s5
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s4
; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s5
; GFX7LESS-NEXT: s_mov_b32 s4, s2
; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: .LBB7_2: ; %atomicrmw.start
@@ -4735,23 +4788,23 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_load_dword s12, s[4:5], 0x34
-; GFX8-NEXT: s_mov_b64 s[6:7], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[4:5], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX8-NEXT: ; implicit-def: $vgpr0
; GFX8-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX8-NEXT: s_cbranch_execz .LBB7_4
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s4, s[2:3], 0x0
-; GFX8-NEXT: s_bcnt1_i32_b64 s5, s[6:7]
+; GFX8-NEXT: s_load_dword s5, s[2:3], 0x0
+; GFX8-NEXT: s_mul_i32 s13, s12, s4
; GFX8-NEXT: s_mov_b64 s[10:11], 0
; GFX8-NEXT: s_mov_b32 s7, 0xf000
-; GFX8-NEXT: s_mul_i32 s13, s12, s5
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
; GFX8-NEXT: s_mov_b32 s6, -1
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v0, s5
; GFX8-NEXT: s_mov_b32 s4, s2
; GFX8-NEXT: s_mov_b32 s5, s3
; GFX8-NEXT: .LBB7_2: ; %atomicrmw.start
@@ -4784,23 +4837,23 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_load_dword s12, s[4:5], 0x34
-; GFX9-NEXT: s_mov_b64 s[6:7], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX9-NEXT: ; implicit-def: $vgpr0
; GFX9-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX9-NEXT: s_cbranch_execz .LBB7_4
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dword s4, s[2:3], 0x0
-; GFX9-NEXT: s_bcnt1_i32_b64 s5, s[6:7]
+; GFX9-NEXT: s_load_dword s5, s[2:3], 0x0
+; GFX9-NEXT: s_mul_i32 s13, s12, s4
; GFX9-NEXT: s_mov_b64 s[10:11], 0
; GFX9-NEXT: s_mov_b32 s7, 0xf000
-; GFX9-NEXT: s_mul_i32 s13, s12, s5
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v0, s5
; GFX9-NEXT: s_mov_b32 s4, s2
; GFX9-NEXT: s_mov_b32 s5, s3
; GFX9-NEXT: .LBB7_2: ; %atomicrmw.start
@@ -4834,25 +4887,25 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1064-NEXT: s_clause 0x1
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064-NEXT: s_load_dword s12, s[4:5], 0x34
-; GFX1064-NEXT: s_mov_b64 s[6:7], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], exec
+; GFX1064-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1064-NEXT: ; implicit-def: $vgpr0
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX1064-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB7_4
; GFX1064-NEXT: ; %bb.1:
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_load_dword s4, s[2:3], 0x0
-; GFX1064-NEXT: s_bcnt1_i32_b64 s5, s[6:7]
+; GFX1064-NEXT: s_load_dword s5, s[2:3], 0x0
+; GFX1064-NEXT: s_mul_i32 s13, s12, s4
; GFX1064-NEXT: s_mov_b64 s[10:11], 0
-; GFX1064-NEXT: s_mul_i32 s13, s12, s5
; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
; GFX1064-NEXT: s_mov_b32 s6, -1
-; GFX1064-NEXT: s_mov_b32 s5, s3
-; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v0, s4
; GFX1064-NEXT: s_mov_b32 s4, s2
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: v_mov_b32_e32 v0, s5
+; GFX1064-NEXT: s_mov_b32 s5, s3
; GFX1064-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: v_mov_b32_e32 v4, v0
@@ -4885,24 +4938,24 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1032-NEXT: s_clause 0x1
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032-NEXT: s_load_dword s8, s[4:5], 0x34
-; GFX1032-NEXT: s_mov_b32 s6, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s4, exec_lo
; GFX1032-NEXT: s_mov_b32 s10, 0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s4, s4
; GFX1032-NEXT: ; implicit-def: $vgpr0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
; GFX1032-NEXT: s_and_saveexec_b32 s9, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB7_4
; GFX1032-NEXT: ; %bb.1:
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dword s4, s[2:3], 0x0
-; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s6
+; GFX1032-NEXT: s_load_dword s5, s[2:3], 0x0
+; GFX1032-NEXT: s_mul_i32 s11, s8, s4
; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1032-NEXT: s_mul_i32 s11, s8, s5
; GFX1032-NEXT: s_mov_b32 s6, -1
-; GFX1032-NEXT: s_mov_b32 s5, s3
-; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mov_b32_e32 v0, s4
; GFX1032-NEXT: s_mov_b32 s4, s2
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: v_mov_b32_e32 v0, s5
+; GFX1032-NEXT: s_mov_b32 s5, s3
; GFX1032-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: v_mov_b32_e32 v4, v0
@@ -4935,26 +4988,26 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1164-NEXT: s_clause 0x1
; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164-NEXT: s_load_b32 s12, s[4:5], 0x34
-; GFX1164-NEXT: s_mov_b64 s[6:7], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: s_mov_b64 s[4:5], exec
; GFX1164-NEXT: s_mov_b64 s[8:9], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1164-NEXT: ; implicit-def: $vgpr0
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB7_4
; GFX1164-NEXT: ; %bb.1:
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_load_b32 s4, s[2:3], 0x0
-; GFX1164-NEXT: s_bcnt1_i32_b64 s5, s[6:7]
+; GFX1164-NEXT: s_load_b32 s5, s[2:3], 0x0
+; GFX1164-NEXT: s_mul_i32 s13, s12, s4
; GFX1164-NEXT: s_mov_b64 s[10:11], 0
-; GFX1164-NEXT: s_mul_i32 s13, s12, s5
; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
; GFX1164-NEXT: s_mov_b32 s6, -1
-; GFX1164-NEXT: s_mov_b32 s5, s3
-; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mov_b32_e32 v0, s4
; GFX1164-NEXT: s_mov_b32 s4, s2
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: v_mov_b32_e32 v0, s5
+; GFX1164-NEXT: s_mov_b32 s5, s3
; GFX1164-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -4991,25 +5044,24 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132-NEXT: s_load_b32 s8, s[4:5], 0x34
-; GFX1132-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s4, exec_lo
; GFX1132-NEXT: s_mov_b32 s10, 0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s4, s4
; GFX1132-NEXT: s_mov_b32 s9, exec_lo
; GFX1132-NEXT: ; implicit-def: $vgpr0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB7_4
; GFX1132-NEXT: ; %bb.1:
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b32 s4, s[2:3], 0x0
-; GFX1132-NEXT: s_bcnt1_i32_b32 s5, s6
+; GFX1132-NEXT: s_load_b32 s5, s[2:3], 0x0
+; GFX1132-NEXT: s_mul_i32 s11, s8, s4
; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1132-NEXT: s_mul_i32 s11, s8, s5
; GFX1132-NEXT: s_mov_b32 s6, -1
-; GFX1132-NEXT: s_mov_b32 s5, s3
-; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: v_mov_b32_e32 v0, s4
; GFX1132-NEXT: s_mov_b32 s4, s2
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: v_mov_b32_e32 v0, s5
+; GFX1132-NEXT: s_mov_b32 s5, s3
; GFX1132-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -5044,33 +5096,32 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1264: ; %bb.0: ; %entry
; GFX1264-NEXT: s_clause 0x1
; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1264-NEXT: s_load_b32 s8, s[4:5], 0x34
-; GFX1264-NEXT: s_mov_b64 s[6:7], exec
+; GFX1264-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1264-NEXT: s_mov_b64 s[4:5], exec
-; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1264-NEXT: ; implicit-def: $vgpr1
-; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX1264-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1264-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
+; GFX1264-NEXT: s_mov_b64 s[4:5], exec
+; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1264-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1264-NEXT: s_cbranch_execz .LBB7_2
; GFX1264-NEXT: ; %bb.1:
-; GFX1264-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1264-NEXT: s_mov_b32 s15, 0x31016000
; GFX1264-NEXT: s_wait_kmcnt 0x0
-; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-NEXT: s_mul_i32 s6, s8, s6
-; GFX1264-NEXT: s_mov_b32 s14, -1
-; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-NEXT: v_mov_b32_e32 v1, s6
-; GFX1264-NEXT: s_mov_b32 s12, s2
-; GFX1264-NEXT: s_mov_b32 s13, s3
-; GFX1264-NEXT: buffer_atomic_sub_u32 v1, off, s[12:15], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1264-NEXT: s_mul_i32 s7, s6, s7
+; GFX1264-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1264-NEXT: v_mov_b32_e32 v1, s7
+; GFX1264-NEXT: s_mov_b32 s10, -1
+; GFX1264-NEXT: s_mov_b32 s8, s2
+; GFX1264-NEXT: s_mov_b32 s9, s3
+; GFX1264-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1264-NEXT: s_wait_loadcnt 0x0
; GFX1264-NEXT: global_inv scope:SCOPE_DEV
; GFX1264-NEXT: .LBB7_2:
; GFX1264-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX1264-NEXT: s_wait_kmcnt 0x0
-; GFX1264-NEXT: v_mul_lo_u32 v0, s8, v0
+; GFX1264-NEXT: v_mul_lo_u32 v0, s6, v0
; GFX1264-NEXT: v_readfirstlane_b32 s2, v1
; GFX1264-NEXT: s_mov_b32 s3, 0x31016000
; GFX1264-NEXT: v_sub_nc_u32_e32 v0, s2, v0
@@ -5083,22 +5134,20 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1232-NEXT: s_clause 0x1
; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1232-NEXT: s_load_b32 s4, s[4:5], 0x34
-; GFX1232-NEXT: s_mov_b32 s6, exec_lo
+; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1232-NEXT: s_mov_b32 s5, exec_lo
-; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1232-NEXT: ; implicit-def: $vgpr1
-; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1232-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: s_bcnt1_i32_b32 s6, s5
+; GFX1232-NEXT: s_mov_b32 s5, exec_lo
; GFX1232-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1232-NEXT: s_cbranch_execz .LBB7_2
; GFX1232-NEXT: ; %bb.1:
-; GFX1232-NEXT: s_bcnt1_i32_b32 s6, s6
-; GFX1232-NEXT: s_mov_b32 s11, 0x31016000
; GFX1232-NEXT: s_wait_kmcnt 0x0
-; GFX1232-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1232-NEXT: s_mul_i32 s6, s4, s6
-; GFX1232-NEXT: s_mov_b32 s10, -1
-; GFX1232-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1232-NEXT: s_mov_b32 s11, 0x31016000
; GFX1232-NEXT: v_mov_b32_e32 v1, s6
+; GFX1232-NEXT: s_mov_b32 s10, -1
; GFX1232-NEXT: s_mov_b32 s8, s2
; GFX1232-NEXT: s_mov_b32 s9, s3
; GFX1232-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -5119,34 +5168,35 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1364: ; %bb.0: ; %entry
; GFX1364-NEXT: s_clause 0x1
; GFX1364-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1364-NEXT: s_load_b32 s8, s[4:5], 0x34 nv
-; GFX1364-NEXT: s_mov_b64 s[6:7], exec
+; GFX1364-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_mov_b64 s[4:5], exec
-; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1364-NEXT: ; implicit-def: $vgpr1
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
+; GFX1364-NEXT: s_mov_b64 s[4:5], exec
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1364-NEXT: s_cbranch_execz .LBB7_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1364-NEXT: s_mov_b32 s15, 0x31016000
; GFX1364-NEXT: s_wait_kmcnt 0x0
-; GFX1364-NEXT: s_mul_i32 s6, s8, s6
-; GFX1364-NEXT: s_mov_b32 s14, -1
-; GFX1364-NEXT: v_mov_b32_e32 v1, s6
-; GFX1364-NEXT: s_mov_b32 s12, s2
-; GFX1364-NEXT: s_mov_b32 s13, s3
+; GFX1364-NEXT: s_mul_i32 s7, s6, s7
+; GFX1364-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1364-NEXT: v_mov_b32_e32 v1, s7
+; GFX1364-NEXT: s_mov_b32 s10, -1
+; GFX1364-NEXT: s_mov_b32 s8, s2
+; GFX1364-NEXT: s_mov_b32 s9, s3
; GFX1364-NEXT: global_wb scope:SCOPE_DEV
; GFX1364-NEXT: s_wait_storecnt 0x0
-; GFX1364-NEXT: buffer_atomic_sub_u32 v1, off, s[12:15], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1364-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1364-NEXT: s_wait_loadcnt 0x0
; GFX1364-NEXT: global_inv scope:SCOPE_DEV
; GFX1364-NEXT: s_wait_loadcnt 0x0
; GFX1364-NEXT: .LBB7_2:
; GFX1364-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX1364-NEXT: s_wait_kmcnt 0x0
-; GFX1364-NEXT: v_mul_lo_u32 v0, s8, v0
+; GFX1364-NEXT: v_mul_lo_u32 v0, s6, v0
; GFX1364-NEXT: v_readfirstlane_b32 s2, v1
; GFX1364-NEXT: s_mov_b32 s3, 0x31016000
; GFX1364-NEXT: v_sub_nc_u32_e32 v0, s2, v0
@@ -5159,20 +5209,20 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1332-NEXT: s_clause 0x1
; GFX1332-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1332-NEXT: s_load_b32 s4, s[4:5], 0x34 nv
-; GFX1332-NEXT: s_mov_b32 s6, exec_lo
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1332-NEXT: s_mov_b32 s5, exec_lo
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1332-NEXT: ; implicit-def: $vgpr1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: s_bcnt1_i32_b32 s6, s5
+; GFX1332-NEXT: s_mov_b32 s5, exec_lo
; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1332-NEXT: s_cbranch_execz .LBB7_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: s_bcnt1_i32_b32 s6, s6
-; GFX1332-NEXT: s_mov_b32 s11, 0x31016000
; GFX1332-NEXT: s_wait_kmcnt 0x0
; GFX1332-NEXT: s_mul_i32 s6, s4, s6
-; GFX1332-NEXT: s_mov_b32 s10, -1
+; GFX1332-NEXT: s_mov_b32 s11, 0x31016000
; GFX1332-NEXT: v_mov_b32_e32 v1, s6
+; GFX1332-NEXT: s_mov_b32 s10, -1
; GFX1332-NEXT: s_mov_b32 s8, s2
; GFX1332-NEXT: s_mov_b32 s9, s3
; GFX1332-NEXT: global_wb scope:SCOPE_DEV
@@ -6446,42 +6496,47 @@ entry:
define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace(1) %inout) {
; GFX7LESS-LABEL: sub_i64_constant:
; GFX7LESS: ; %bb.0: ; %entry
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[0:1]
+; GFX7LESS-NEXT: v_mul_hi_u32 v0, 5, s6
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v4, s7, v0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v4, exec_hi, v1
+; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v0
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX7LESS-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB9_4
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX7LESS-NEXT: s_mov_b64 s[10:11], 0
-; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT: s_load_dwordx2 s[14:15], s[2:3], 0x0
+; GFX7LESS-NEXT: s_mul_i32 s5, s6, 0
+; GFX7LESS-NEXT: s_add_u32 s4, s4, s5
; GFX7LESS-NEXT: s_mul_i32 s12, s6, 5
+; GFX7LESS-NEXT: s_mov_b64 s[10:11], 0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s4
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s5
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s14
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s15
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, s4
+; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s6, -1
; GFX7LESS-NEXT: s_mov_b32 s4, s2
; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v0
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v1
-; GFX7LESS-NEXT: v_subrev_i32_e32 v5, vcc, s12, v7
-; GFX7LESS-NEXT: v_subbrev_u32_e32 v6, vcc, 0, v8, vcc
+; GFX7LESS-NEXT: v_mov_b32_e32 v8, v0
+; GFX7LESS-NEXT: v_mov_b32_e32 v9, v1
+; GFX7LESS-NEXT: v_subrev_i32_e32 v6, vcc, s12, v8
+; GFX7LESS-NEXT: v_subb_u32_e32 v7, vcc, v9, v5, vcc
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, v5
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v6
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v7
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v8
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, v6
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v7
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v8
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v9
; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS-NEXT: buffer_wbinvl1
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX7LESS-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[10:11]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB9_2
@@ -6505,41 +6560,46 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX8-LABEL: sub_i64_constant:
; GFX8: ; %bb.0: ; %entry
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[0:1]
+; GFX8-NEXT: v_mul_hi_u32 v0, 5, s6
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b64 s[6:7], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v1
+; GFX8-NEXT: v_readfirstlane_b32 s4, v0
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX8-NEXT: s_cbranch_execz .LBB9_4
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
-; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX8-NEXT: s_mov_b64 s[10:11], 0
-; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_load_dwordx2 s[14:15], s[2:3], 0x0
+; GFX8-NEXT: s_mul_i32 s5, s6, 0
+; GFX8-NEXT: s_add_u32 s4, s4, s5
; GFX8-NEXT: s_mul_i32 s12, s6, 5
+; GFX8-NEXT: s_mov_b64 s[10:11], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_mov_b32_e32 v0, s14
+; GFX8-NEXT: v_mov_b32_e32 v1, s15
+; GFX8-NEXT: v_mov_b32_e32 v5, s4
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
; GFX8-NEXT: s_mov_b32 s6, -1
; GFX8-NEXT: s_mov_b32 s4, s2
; GFX8-NEXT: s_mov_b32 s5, s3
; GFX8-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_mov_b32_e32 v7, v0
-; GFX8-NEXT: v_mov_b32_e32 v8, v1
-; GFX8-NEXT: v_subrev_u32_e32 v5, vcc, s12, v7
-; GFX8-NEXT: v_subbrev_u32_e32 v6, vcc, 0, v8, vcc
-; GFX8-NEXT: v_mov_b32_e32 v0, v5
-; GFX8-NEXT: v_mov_b32_e32 v1, v6
-; GFX8-NEXT: v_mov_b32_e32 v2, v7
-; GFX8-NEXT: v_mov_b32_e32 v3, v8
+; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_subrev_u32_e32 v6, vcc, s12, v8
+; GFX8-NEXT: v_subb_u32_e32 v7, vcc, v9, v5, vcc
+; GFX8-NEXT: v_mov_b32_e32 v0, v6
+; GFX8-NEXT: v_mov_b32_e32 v1, v7
+; GFX8-NEXT: v_mov_b32_e32 v2, v8
+; GFX8-NEXT: v_mov_b32_e32 v3, v9
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX8-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[10:11]
; GFX8-NEXT: s_cbranch_execnz .LBB9_2
@@ -6563,40 +6623,44 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX9-LABEL: sub_i64_constant:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b64 s[6:7], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX9-NEXT: s_cbranch_execz .LBB9_4
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
-; GFX9-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
+; GFX9-NEXT: s_mul_i32 s12, s4, 5
+; GFX9-NEXT: s_mul_hi_u32 s5, 5, s4
+; GFX9-NEXT: s_mul_i32 s4, s4, 0
+; GFX9-NEXT: s_add_u32 s4, s5, s4
; GFX9-NEXT: s_mov_b64 s[10:11], 0
-; GFX9-NEXT: s_mov_b32 s7, 0xf000
-; GFX9-NEXT: s_mul_i32 s12, s6, 5
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-NEXT: v_mov_b32_e32 v5, s4
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
; GFX9-NEXT: s_mov_b32 s6, -1
; GFX9-NEXT: s_mov_b32 s4, s2
; GFX9-NEXT: s_mov_b32 s5, s3
; GFX9-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_mov_b32_e32 v7, v0
-; GFX9-NEXT: v_mov_b32_e32 v8, v1
-; GFX9-NEXT: v_subrev_co_u32_e32 v5, vcc, s12, v7
-; GFX9-NEXT: v_subbrev_co_u32_e32 v6, vcc, 0, v8, vcc
-; GFX9-NEXT: v_mov_b32_e32 v0, v5
-; GFX9-NEXT: v_mov_b32_e32 v1, v6
-; GFX9-NEXT: v_mov_b32_e32 v2, v7
-; GFX9-NEXT: v_mov_b32_e32 v3, v8
+; GFX9-NEXT: v_mov_b32_e32 v8, v0
+; GFX9-NEXT: v_mov_b32_e32 v9, v1
+; GFX9-NEXT: v_subrev_co_u32_e32 v6, vcc, s12, v8
+; GFX9-NEXT: v_subb_co_u32_e32 v7, vcc, v9, v5, vcc
+; GFX9-NEXT: v_mov_b32_e32 v0, v6
+; GFX9-NEXT: v_mov_b32_e32 v1, v7
+; GFX9-NEXT: v_mov_b32_e32 v2, v8
+; GFX9-NEXT: v_mov_b32_e32 v3, v9
; GFX9-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX9-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[10:11]
; GFX9-NEXT: s_cbranch_execnz .LBB9_2
@@ -6620,32 +6684,35 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1064-LABEL: sub_i64_constant:
; GFX1064: ; %bb.0: ; %entry
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1064-NEXT: s_mov_b64 s[6:7], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], exec
+; GFX1064-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1064-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX1064-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB9_4
; GFX1064-NEXT: ; %bb.1:
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
-; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
+; GFX1064-NEXT: s_mul_i32 s12, s4, 5
+; GFX1064-NEXT: s_mul_hi_u32 s5, 5, s4
+; GFX1064-NEXT: s_mul_i32 s4, s4, 0
; GFX1064-NEXT: s_mov_b64 s[10:11], 0
-; GFX1064-NEXT: s_mul_i32 s12, s6, 5
-; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1064-NEXT: s_mov_b32 s6, -1
-; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v0, s4
-; GFX1064-NEXT: v_mov_b32_e32 v1, s5
+; GFX1064-NEXT: s_add_u32 s13, s5, s4
; GFX1064-NEXT: s_mov_b32 s4, s2
; GFX1064-NEXT: s_mov_b32 s5, s3
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: v_mov_b32_e32 v0, s6
+; GFX1064-NEXT: v_mov_b32_e32 v1, s7
+; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1064-NEXT: s_mov_b32 s6, -1
; GFX1064-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: v_mov_b32_e32 v7, v0
; GFX1064-NEXT: v_mov_b32_e32 v8, v1
; GFX1064-NEXT: v_sub_co_u32 v5, vcc, v7, s12
-; GFX1064-NEXT: v_subrev_co_ci_u32_e32 v6, vcc, 0, v8, vcc
+; GFX1064-NEXT: v_subrev_co_ci_u32_e32 v6, vcc, s13, v8, vcc
; GFX1064-NEXT: v_mov_b32_e32 v2, v7
; GFX1064-NEXT: v_mov_b32_e32 v0, v5
; GFX1064-NEXT: v_mov_b32_e32 v3, v8
@@ -6677,31 +6744,34 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1032-LABEL: sub_i64_constant:
; GFX1032: ; %bb.0: ; %entry
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1032-NEXT: s_mov_b32 s6, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s4, exec_lo
; GFX1032-NEXT: s_mov_b32 s9, 0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s4, s4
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v4
; GFX1032-NEXT: s_and_saveexec_b32 s8, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB9_4
; GFX1032-NEXT: ; %bb.1:
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
-; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s6
-; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1032-NEXT: s_mul_i32 s10, s6, 5
-; GFX1032-NEXT: s_mov_b32 s6, -1
-; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mov_b32_e32 v0, s4
-; GFX1032-NEXT: v_mov_b32_e32 v1, s5
+; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
+; GFX1032-NEXT: s_mul_hi_u32 s5, 5, s4
+; GFX1032-NEXT: s_mul_i32 s11, s4, 0
+; GFX1032-NEXT: s_mul_i32 s10, s4, 5
+; GFX1032-NEXT: s_add_u32 s11, s5, s11
; GFX1032-NEXT: s_mov_b32 s4, s2
; GFX1032-NEXT: s_mov_b32 s5, s3
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: v_mov_b32_e32 v0, s6
+; GFX1032-NEXT: v_mov_b32_e32 v1, s7
+; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT: s_mov_b32 s6, -1
; GFX1032-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: v_mov_b32_e32 v7, v0
; GFX1032-NEXT: v_mov_b32_e32 v8, v1
; GFX1032-NEXT: v_sub_co_u32 v5, vcc_lo, v7, s10
-; GFX1032-NEXT: v_subrev_co_ci_u32_e32 v6, vcc_lo, 0, v8, vcc_lo
+; GFX1032-NEXT: v_subrev_co_ci_u32_e32 v6, vcc_lo, s11, v8, vcc_lo
; GFX1032-NEXT: v_mov_b32_e32 v2, v7
; GFX1032-NEXT: v_mov_b32_e32 v0, v5
; GFX1032-NEXT: v_mov_b32_e32 v3, v8
@@ -6733,27 +6803,30 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1164-LABEL: sub_i64_constant:
; GFX1164: ; %bb.0: ; %entry
; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1164-NEXT: s_mov_b64 s[6:7], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: s_mov_b64 s[4:5], exec
; GFX1164-NEXT: s_mov_b64 s[8:9], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1164-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-NEXT: s_cbranch_execz .LBB9_4
; GFX1164-NEXT: ; %bb.1:
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
-; GFX1164-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1164-NEXT: s_load_b64 s[6:7], s[2:3], 0x0
+; GFX1164-NEXT: s_mul_i32 s12, s4, 5
+; GFX1164-NEXT: s_mul_hi_u32 s5, 5, s4
+; GFX1164-NEXT: s_mul_i32 s4, s4, 0
; GFX1164-NEXT: s_mov_b64 s[10:11], 0
-; GFX1164-NEXT: s_mul_i32 s12, s6, 5
-; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1164-NEXT: s_mov_b32 s6, -1
-; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mov_b32_e32 v0, s4
-; GFX1164-NEXT: v_mov_b32_e32 v1, s5
+; GFX1164-NEXT: s_add_u32 s13, s5, s4
; GFX1164-NEXT: s_mov_b32 s4, s2
; GFX1164-NEXT: s_mov_b32 s5, s3
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: v_mov_b32_e32 v0, s6
+; GFX1164-NEXT: v_mov_b32_e32 v1, s7
+; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1164-NEXT: s_mov_b32 s6, -1
; GFX1164-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
@@ -6761,7 +6834,7 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1164-NEXT: v_mov_b32_e32 v8, v1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: v_sub_co_u32 v5, vcc, v7, s12
-; GFX1164-NEXT: v_subrev_co_ci_u32_e64 v6, null, 0, v8, vcc
+; GFX1164-NEXT: v_subrev_co_ci_u32_e64 v6, null, s13, v8, vcc
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX1164-NEXT: v_mov_b32_e32 v0, v5
; GFX1164-NEXT: v_mov_b32_e32 v2, v7
@@ -6798,32 +6871,34 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1132-LABEL: sub_i64_constant:
; GFX1132: ; %bb.0: ; %entry
; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1132-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s4, exec_lo
; GFX1132-NEXT: s_mov_b32 s9, 0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s4, s4
; GFX1132-NEXT: s_mov_b32 s8, exec_lo
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-NEXT: s_cbranch_execz .LBB9_4
; GFX1132-NEXT: ; %bb.1:
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
-; GFX1132-NEXT: s_bcnt1_i32_b32 s6, s6
-; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1132-NEXT: s_mul_i32 s10, s6, 5
-; GFX1132-NEXT: s_mov_b32 s6, -1
-; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
+; GFX1132-NEXT: s_load_b64 s[6:7], s[2:3], 0x0
+; GFX1132-NEXT: s_mul_hi_u32 s5, 5, s4
+; GFX1132-NEXT: s_mul_i32 s11, s4, 0
+; GFX1132-NEXT: s_mul_i32 s10, s4, 5
+; GFX1132-NEXT: s_add_u32 s11, s5, s11
; GFX1132-NEXT: s_mov_b32 s4, s2
; GFX1132-NEXT: s_mov_b32 s5, s3
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
+; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-NEXT: s_mov_b32 s6, -1
; GFX1132-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v7, v0
; GFX1132-NEXT: v_sub_co_u32 v5, vcc_lo, v7, s10
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1132-NEXT: v_subrev_co_ci_u32_e64 v6, null, 0, v8, vcc_lo
+; GFX1132-NEXT: v_subrev_co_ci_u32_e64 v6, null, s11, v8, vcc_lo
; GFX1132-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
; GFX1132-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
; GFX1132-NEXT: buffer_atomic_cmpswap_b64 v[0:3], off, s[4:7], 0 glc
@@ -6856,22 +6931,24 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1264-LABEL: sub_i64_constant:
; GFX1264: ; %bb.0: ; %entry
; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1264-NEXT: s_mov_b64 s[6:7], exec
+; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1264-NEXT: s_mov_b64 s[4:5], exec
+; GFX1264-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1264-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX1264-NEXT: s_mov_b64 s[4:5], exec
-; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1264-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1264-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1264-NEXT: s_cbranch_execz .LBB9_2
; GFX1264-NEXT: ; %bb.1:
-; GFX1264-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1264-NEXT: v_mov_b32_e32 v1, 0
-; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1264-NEXT: s_mul_hi_u32 s7, 5, s6
+; GFX1264-NEXT: s_mul_i32 s8, s6, 0
; GFX1264-NEXT: s_mul_i32 s6, s6, 5
-; GFX1264-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1264-NEXT: s_add_co_u32 s7, s7, s8
; GFX1264-NEXT: v_mov_b32_e32 v0, s6
+; GFX1264-NEXT: v_mov_b32_e32 v1, s7
+; GFX1264-NEXT: s_mov_b32 s11, 0x31016000
; GFX1264-NEXT: s_mov_b32 s10, -1
; GFX1264-NEXT: s_wait_kmcnt 0x0
; GFX1264-NEXT: s_mov_b32 s8, s2
@@ -6897,19 +6974,23 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1232-LABEL: sub_i64_constant:
; GFX1232: ; %bb.0: ; %entry
; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1232-NEXT: s_mov_b32 s6, exec_lo
+; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1232-NEXT: s_mov_b32 s4, exec_lo
-; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
; GFX1232-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1232-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: s_bcnt1_i32_b32 s5, s4
+; GFX1232-NEXT: s_mov_b32 s4, exec_lo
; GFX1232-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1232-NEXT: s_cbranch_execz .LBB9_2
; GFX1232-NEXT: ; %bb.1:
-; GFX1232-NEXT: s_bcnt1_i32_b32 s5, s6
+; GFX1232-NEXT: s_mul_hi_u32 s7, 5, s5
+; GFX1232-NEXT: s_mul_i32 s8, s5, 0
+; GFX1232-NEXT: s_mul_i32 s6, s5, 5
+; GFX1232-NEXT: s_add_co_u32 s7, s7, s8
+; GFX1232-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1232-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
; GFX1232-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1232-NEXT: s_mul_i32 s5, s5, 5
; GFX1232-NEXT: s_mov_b32 s10, -1
-; GFX1232-NEXT: v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v1, 0
; GFX1232-NEXT: s_wait_kmcnt 0x0
; GFX1232-NEXT: s_mov_b32 s8, s2
; GFX1232-NEXT: s_mov_b32 s9, s3
@@ -6934,20 +7015,24 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1364-LABEL: sub_i64_constant:
; GFX1364: ; %bb.0: ; %entry
; GFX1364-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1364-NEXT: s_mov_b64 s[6:7], exec
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_mov_b64 s[4:5], exec
-; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1364-NEXT: s_mov_b64 s[4:5], exec
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1364-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1364-NEXT: s_cbranch_execz .LBB9_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1364-NEXT: v_mov_b32_e32 v1, 0
+; GFX1364-NEXT: s_mul_hi_u32 s7, 5, s6
+; GFX1364-NEXT: s_mul_i32 s8, s6, 0
; GFX1364-NEXT: s_mul_i32 s6, s6, 5
-; GFX1364-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1364-NEXT: s_add_co_u32 s7, s7, s8
; GFX1364-NEXT: v_mov_b32_e32 v0, s6
+; GFX1364-NEXT: v_mov_b32_e32 v1, s7
+; GFX1364-NEXT: s_mov_b32 s11, 0x31016000
; GFX1364-NEXT: s_mov_b32 s10, -1
; GFX1364-NEXT: s_wait_kmcnt 0x0
; GFX1364-NEXT: s_mov_b32 s8, s2
@@ -6976,19 +7061,23 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1332-LABEL: sub_i64_constant:
; GFX1332: ; %bb.0: ; %entry
; GFX1332-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1332-NEXT: s_mov_b32 s6, exec_lo
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1332-NEXT: s_mov_b32 s4, exec_lo
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
; GFX1332-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: s_bcnt1_i32_b32 s5, s4
+; GFX1332-NEXT: s_mov_b32 s4, exec_lo
; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1332-NEXT: s_cbranch_execz .LBB9_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: s_bcnt1_i32_b32 s5, s6
+; GFX1332-NEXT: s_mul_hi_u32 s7, 5, s5
+; GFX1332-NEXT: s_mul_i32 s8, s5, 0
+; GFX1332-NEXT: s_mul_i32 s6, s5, 5
+; GFX1332-NEXT: s_add_co_u32 s7, s7, s8
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1332-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
; GFX1332-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1332-NEXT: s_mul_i32 s5, s5, 5
; GFX1332-NEXT: s_mov_b32 s10, -1
-; GFX1332-NEXT: v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v1, 0
; GFX1332-NEXT: s_wait_kmcnt 0x0
; GFX1332-NEXT: s_mov_b32 s8, s2
; GFX1332-NEXT: s_mov_b32 s9, s3
@@ -7021,48 +7110,49 @@ entry:
define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(1) %inout, i64 %subitive) {
; GFX7LESS-LABEL: sub_i64_uniform:
; GFX7LESS: ; %bb.0: ; %entry
-; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX7LESS-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
-; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v4, s7, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[0:1]
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6
+; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mul_hi_u32 v0, s8, v0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v4, exec_hi, v1
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
+; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v0
; GFX7LESS-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[10:11], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB10_4
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
-; GFX7LESS-NEXT: v_mul_hi_u32 v0, s8, v0
-; GFX7LESS-NEXT: s_mul_i32 s7, s9, s6
-; GFX7LESS-NEXT: s_mul_i32 s6, s8, s6
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, s6
-; GFX7LESS-NEXT: v_add_i32_e32 v5, vcc, s7, v0
+; GFX7LESS-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0
+; GFX7LESS-NEXT: s_mul_i32 s5, s9, s6
+; GFX7LESS-NEXT: s_add_u32 s4, s4, s5
+; GFX7LESS-NEXT: s_mul_i32 s14, s8, s6
; GFX7LESS-NEXT: s_mov_b64 s[12:13], 0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s4
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s5
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s16
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s17
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, s4
; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s6, -1
; GFX7LESS-NEXT: s_mov_b32 s4, s2
; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v0
-; GFX7LESS-NEXT: v_mov_b32_e32 v10, v1
-; GFX7LESS-NEXT: v_sub_i32_e32 v7, vcc, v9, v6
-; GFX7LESS-NEXT: v_subb_u32_e32 v8, vcc, v10, v5, vcc
+; GFX7LESS-NEXT: v_mov_b32_e32 v8, v0
+; GFX7LESS-NEXT: v_mov_b32_e32 v9, v1
+; GFX7LESS-NEXT: v_subrev_i32_e32 v6, vcc, s14, v8
+; GFX7LESS-NEXT: v_subb_u32_e32 v7, vcc, v9, v5, vcc
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, v7
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v8
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v9
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v10
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, v6
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v7
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v8
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v9
; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS-NEXT: buffer_wbinvl1
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX7LESS-NEXT: s_or_b64 s[12:13], vcc, s[12:13]
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[12:13]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB10_2
@@ -7071,7 +7161,7 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX7LESS-NEXT: .LBB10_4: ; %Flow4
; GFX7LESS-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v1
-; GFX7LESS-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT: s_waitcnt expcnt(0)
; GFX7LESS-NEXT: v_mul_lo_u32 v1, s9, v4
; GFX7LESS-NEXT: v_mul_hi_u32 v2, s8, v4
; GFX7LESS-NEXT: v_readfirstlane_b32 s5, v0
@@ -7087,45 +7177,48 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX8-LABEL: sub_i64_uniform:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34
-; GFX8-NEXT: s_mov_b64 s[6:7], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v6, s7, v0
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v6
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s6
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mul_hi_u32 v0, s8, v0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
+; GFX8-NEXT: v_readfirstlane_b32 s4, v0
; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[10:11], vcc
; GFX8-NEXT: s_cbranch_execz .LBB10_4
; GFX8-NEXT: ; %bb.1:
-; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v0, s6
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mad_u64_u32 v[4:5], s[4:5], s8, v0, 0
-; GFX8-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
-; GFX8-NEXT: s_mul_i32 s6, s9, s6
+; GFX8-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0
+; GFX8-NEXT: s_mul_i32 s5, s9, s6
+; GFX8-NEXT: s_add_u32 s4, s4, s5
+; GFX8-NEXT: s_mul_i32 s14, s8, s6
; GFX8-NEXT: s_mov_b64 s[12:13], 0
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, s6, v5
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_mov_b32_e32 v0, s16
+; GFX8-NEXT: v_mov_b32_e32 v1, s17
+; GFX8-NEXT: v_mov_b32_e32 v5, s4
; GFX8-NEXT: s_mov_b32 s7, 0xf000
; GFX8-NEXT: s_mov_b32 s6, -1
; GFX8-NEXT: s_mov_b32 s4, s2
; GFX8-NEXT: s_mov_b32 s5, s3
; GFX8-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_mov_b32_e32 v9, v0
-; GFX8-NEXT: v_mov_b32_e32 v10, v1
-; GFX8-NEXT: v_sub_u32_e32 v7, vcc, v9, v4
-; GFX8-NEXT: v_subb_u32_e32 v8, vcc, v10, v5, vcc
-; GFX8-NEXT: v_mov_b32_e32 v0, v7
-; GFX8-NEXT: v_mov_b32_e32 v1, v8
-; GFX8-NEXT: v_mov_b32_e32 v2, v9
-; GFX8-NEXT: v_mov_b32_e32 v3, v10
+; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_subrev_u32_e32 v6, vcc, s14, v8
+; GFX8-NEXT: v_subb_u32_e32 v7, vcc, v9, v5, vcc
+; GFX8-NEXT: v_mov_b32_e32 v0, v6
+; GFX8-NEXT: v_mov_b32_e32 v1, v7
+; GFX8-NEXT: v_mov_b32_e32 v2, v8
+; GFX8-NEXT: v_mov_b32_e32 v3, v9
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX8-NEXT: s_or_b64 s[12:13], vcc, s[12:13]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[12:13]
; GFX8-NEXT: s_cbranch_execnz .LBB10_2
@@ -7133,13 +7226,12 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX8-NEXT: s_or_b64 exec, exec, s[12:13]
; GFX8-NEXT: .LBB10_4: ; %Flow4
; GFX8-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mul_lo_u32 v4, s9, v6
-; GFX8-NEXT: v_mad_u64_u32 v[2:3], s[4:5], s8, v6, 0
+; GFX8-NEXT: v_mul_lo_u32 v5, s9, v4
+; GFX8-NEXT: v_mad_u64_u32 v[2:3], s[4:5], s8, v4, 0
; GFX8-NEXT: v_readfirstlane_b32 s4, v1
; GFX8-NEXT: v_readfirstlane_b32 s5, v0
; GFX8-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, v3, v4
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v3, v5
; GFX8-NEXT: v_mov_b32_e32 v3, s4
; GFX8-NEXT: v_sub_u32_e32 v0, vcc, s5, v2
; GFX8-NEXT: s_mov_b32 s2, -1
@@ -7151,26 +7243,26 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34
-; GFX9-NEXT: s_mov_b64 s[6:7], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[10:11], vcc
; GFX9-NEXT: s_cbranch_execz .LBB10_4
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
-; GFX9-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX9-NEXT: s_mul_i32 s7, s9, s6
-; GFX9-NEXT: s_mul_hi_u32 s12, s8, s6
-; GFX9-NEXT: s_add_i32 s7, s12, s7
-; GFX9-NEXT: s_mul_i32 s14, s8, s6
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
+; GFX9-NEXT: s_mul_i32 s14, s8, s4
+; GFX9-NEXT: s_mul_hi_u32 s5, s8, s4
+; GFX9-NEXT: s_mul_i32 s4, s9, s4
+; GFX9-NEXT: s_add_u32 s4, s5, s4
; GFX9-NEXT: s_mov_b64 s[12:13], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-NEXT: v_mov_b32_e32 v1, s5
-; GFX9-NEXT: v_mov_b32_e32 v5, s7
+; GFX9-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-NEXT: v_mov_b32_e32 v5, s4
; GFX9-NEXT: s_mov_b32 s7, 0xf000
; GFX9-NEXT: s_mov_b32 s6, -1
; GFX9-NEXT: s_mov_b32 s4, s2
@@ -7214,29 +7306,29 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1064-NEXT: s_clause 0x1
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34
-; GFX1064-NEXT: s_mov_b64 s[6:7], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], exec
+; GFX1064-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1064-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX1064-NEXT: s_and_saveexec_b64 s[10:11], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB10_4
; GFX1064-NEXT: ; %bb.1:
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
-; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
+; GFX1064-NEXT: s_mul_i32 s14, s8, s4
+; GFX1064-NEXT: s_mul_hi_u32 s5, s8, s4
+; GFX1064-NEXT: s_mul_i32 s4, s9, s4
; GFX1064-NEXT: s_mov_b64 s[12:13], 0
-; GFX1064-NEXT: s_mul_i32 s7, s9, s6
-; GFX1064-NEXT: s_mul_hi_u32 s15, s8, s6
-; GFX1064-NEXT: s_mul_i32 s14, s8, s6
-; GFX1064-NEXT: s_add_i32 s15, s15, s7
-; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1064-NEXT: s_mov_b32 s6, -1
-; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v0, s4
-; GFX1064-NEXT: v_mov_b32_e32 v1, s5
+; GFX1064-NEXT: s_add_u32 s15, s5, s4
; GFX1064-NEXT: s_mov_b32 s4, s2
; GFX1064-NEXT: s_mov_b32 s5, s3
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: v_mov_b32_e32 v0, s6
+; GFX1064-NEXT: v_mov_b32_e32 v1, s7
+; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1064-NEXT: s_mov_b32 s6, -1
; GFX1064-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: v_mov_b32_e32 v7, v0
@@ -7276,28 +7368,28 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1032-NEXT: s_clause 0x1
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34
-; GFX1032-NEXT: s_mov_b32 s6, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s4, exec_lo
; GFX1032-NEXT: s_mov_b32 s11, 0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s4, s4
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v4
; GFX1032-NEXT: s_and_saveexec_b32 s10, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB10_4
; GFX1032-NEXT: ; %bb.1:
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
-; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s6
-; GFX1032-NEXT: s_mul_i32 s7, s9, s6
-; GFX1032-NEXT: s_mul_hi_u32 s13, s8, s6
-; GFX1032-NEXT: s_mul_i32 s12, s8, s6
-; GFX1032-NEXT: s_add_i32 s13, s13, s7
-; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1032-NEXT: s_mov_b32 s6, -1
-; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mov_b32_e32 v0, s4
-; GFX1032-NEXT: v_mov_b32_e32 v1, s5
+; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
+; GFX1032-NEXT: s_mul_hi_u32 s5, s8, s4
+; GFX1032-NEXT: s_mul_i32 s13, s9, s4
+; GFX1032-NEXT: s_mul_i32 s12, s8, s4
+; GFX1032-NEXT: s_add_u32 s13, s5, s13
; GFX1032-NEXT: s_mov_b32 s4, s2
; GFX1032-NEXT: s_mov_b32 s5, s3
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: v_mov_b32_e32 v0, s6
+; GFX1032-NEXT: v_mov_b32_e32 v1, s7
+; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT: s_mov_b32 s6, -1
; GFX1032-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: v_mov_b32_e32 v7, v0
@@ -7337,30 +7429,30 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1164-NEXT: s_clause 0x1
; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164-NEXT: s_load_b64 s[8:9], s[4:5], 0x34
-; GFX1164-NEXT: s_mov_b64 s[6:7], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: s_mov_b64 s[4:5], exec
; GFX1164-NEXT: s_mov_b64 s[10:11], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1164-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-NEXT: s_cbranch_execz .LBB10_4
; GFX1164-NEXT: ; %bb.1:
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
-; GFX1164-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1164-NEXT: s_load_b64 s[6:7], s[2:3], 0x0
+; GFX1164-NEXT: s_mul_i32 s14, s8, s4
+; GFX1164-NEXT: s_mul_hi_u32 s5, s8, s4
+; GFX1164-NEXT: s_mul_i32 s4, s9, s4
; GFX1164-NEXT: s_mov_b64 s[12:13], 0
-; GFX1164-NEXT: s_mul_i32 s7, s9, s6
-; GFX1164-NEXT: s_mul_hi_u32 s15, s8, s6
-; GFX1164-NEXT: s_mul_i32 s14, s8, s6
-; GFX1164-NEXT: s_add_i32 s15, s15, s7
-; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1164-NEXT: s_mov_b32 s6, -1
-; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mov_b32_e32 v0, s4
-; GFX1164-NEXT: v_mov_b32_e32 v1, s5
+; GFX1164-NEXT: s_add_u32 s15, s5, s4
; GFX1164-NEXT: s_mov_b32 s4, s2
; GFX1164-NEXT: s_mov_b32 s5, s3
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: v_mov_b32_e32 v0, s6
+; GFX1164-NEXT: v_mov_b32_e32 v1, s7
+; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1164-NEXT: s_mov_b32 s6, -1
; GFX1164-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
@@ -7407,29 +7499,27 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132-NEXT: s_load_b64 s[8:9], s[4:5], 0x34
-; GFX1132-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s4, exec_lo
; GFX1132-NEXT: s_mov_b32 s11, 0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s4, s4
; GFX1132-NEXT: s_mov_b32 s10, exec_lo
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-NEXT: s_cbranch_execz .LBB10_4
; GFX1132-NEXT: ; %bb.1:
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
-; GFX1132-NEXT: s_bcnt1_i32_b32 s6, s6
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_mul_i32 s7, s9, s6
-; GFX1132-NEXT: s_mul_hi_u32 s13, s8, s6
-; GFX1132-NEXT: s_mul_i32 s12, s8, s6
-; GFX1132-NEXT: s_add_i32 s13, s13, s7
-; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1132-NEXT: s_mov_b32 s6, -1
-; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
+; GFX1132-NEXT: s_load_b64 s[6:7], s[2:3], 0x0
+; GFX1132-NEXT: s_mul_hi_u32 s5, s8, s4
+; GFX1132-NEXT: s_mul_i32 s13, s9, s4
+; GFX1132-NEXT: s_mul_i32 s12, s8, s4
+; GFX1132-NEXT: s_add_u32 s13, s5, s13
; GFX1132-NEXT: s_mov_b32 s4, s2
; GFX1132-NEXT: s_mov_b32 s5, s3
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
+; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-NEXT: s_mov_b32 s6, -1
; GFX1132-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -7471,23 +7561,25 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1264-NEXT: s_clause 0x1
; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1264-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
-; GFX1264-NEXT: s_mov_b64 s[8:9], exec
-; GFX1264-NEXT: s_mov_b32 s11, 0
-; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, s8, 0
+; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1264-NEXT: s_mov_b64 s[6:7], exec
-; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v2, s9, v0
+; GFX1264-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1264-NEXT: s_bcnt1_i32_b64 s8, s[6:7]
+; GFX1264-NEXT: s_mov_b64 s[6:7], exec
+; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1264-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1264-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1264-NEXT: s_cbranch_execz .LBB10_2
; GFX1264-NEXT: ; %bb.1:
-; GFX1264-NEXT: s_bcnt1_i32_b64 s10, s[8:9]
; GFX1264-NEXT: s_wait_kmcnt 0x0
-; GFX1264-NEXT: s_mul_u64 s[8:9], s[4:5], s[10:11]
-; GFX1264-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1264-NEXT: s_mul_hi_u32 s9, s4, s8
+; GFX1264-NEXT: s_mul_i32 s10, s5, s8
+; GFX1264-NEXT: s_mul_i32 s8, s4, s8
+; GFX1264-NEXT: s_add_co_u32 s9, s9, s10
; GFX1264-NEXT: v_mov_b32_e32 v0, s8
; GFX1264-NEXT: v_mov_b32_e32 v1, s9
+; GFX1264-NEXT: s_mov_b32 s11, 0x31016000
; GFX1264-NEXT: s_mov_b32 s10, -1
; GFX1264-NEXT: s_mov_b32 s8, s2
; GFX1264-NEXT: s_mov_b32 s9, s3
@@ -7514,30 +7606,31 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1232-NEXT: s_clause 0x1
; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1232-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1232-NEXT: s_mov_b32 s6, exec_lo
-; GFX1232-NEXT: s_mov_b32 s7, 0
-; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
-; GFX1232-NEXT: s_mov_b32 s8, exec_lo
; GFX1232-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1232-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: s_bcnt1_i32_b32 s7, s6
+; GFX1232-NEXT: s_mov_b32 s6, exec_lo
; GFX1232-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1232-NEXT: s_cbranch_execz .LBB10_2
; GFX1232-NEXT: ; %bb.1:
-; GFX1232-NEXT: s_bcnt1_i32_b32 s6, s6
-; GFX1232-NEXT: s_mov_b32 s15, 0x31016000
; GFX1232-NEXT: s_wait_kmcnt 0x0
-; GFX1232-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232-NEXT: s_mul_u64 s[6:7], s[4:5], s[6:7]
-; GFX1232-NEXT: s_mov_b32 s14, -1
-; GFX1232-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
-; GFX1232-NEXT: s_mov_b32 s12, s2
-; GFX1232-NEXT: s_mov_b32 s13, s3
-; GFX1232-NEXT: buffer_atomic_sub_u64 v[0:1], off, s[12:15], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1232-NEXT: s_mul_hi_u32 s9, s4, s7
+; GFX1232-NEXT: s_mul_i32 s10, s5, s7
+; GFX1232-NEXT: s_mul_i32 s8, s4, s7
+; GFX1232-NEXT: s_add_co_u32 s9, s9, s10
+; GFX1232-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1232-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
+; GFX1232-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1232-NEXT: s_mov_b32 s10, -1
+; GFX1232-NEXT: s_mov_b32 s8, s2
+; GFX1232-NEXT: s_mov_b32 s9, s3
+; GFX1232-NEXT: buffer_atomic_sub_u64 v[0:1], off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1232-NEXT: s_wait_loadcnt 0x0
; GFX1232-NEXT: global_inv scope:SCOPE_DEV
; GFX1232-NEXT: .LBB10_2:
-; GFX1232-NEXT: s_or_b32 exec_lo, exec_lo, s8
+; GFX1232-NEXT: s_or_b32 exec_lo, exec_lo, s6
; GFX1232-NEXT: s_wait_kmcnt 0x0
; GFX1232-NEXT: v_mad_co_u64_u32 v[3:4], null, s4, v2, 0
; GFX1232-NEXT: v_readfirstlane_b32 s2, v0
@@ -7556,22 +7649,25 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1364-NEXT: s_clause 0x1
; GFX1364-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1364-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
-; GFX1364-NEXT: s_mov_b64 s[8:9], exec
-; GFX1364-NEXT: s_mov_b32 s11, 0
-; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, s8, 0
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_mov_b64 s[6:7], exec
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v2, s9, v0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_bcnt1_i32_b64 s8, s[6:7]
+; GFX1364-NEXT: s_mov_b64 s[6:7], exec
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1364-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1364-NEXT: s_cbranch_execz .LBB10_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: s_bcnt1_i32_b64 s10, s[8:9]
; GFX1364-NEXT: s_wait_kmcnt 0x0
-; GFX1364-NEXT: s_mul_u64 s[8:9], s[4:5], s[10:11]
-; GFX1364-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1364-NEXT: s_mul_hi_u32 s9, s4, s8
+; GFX1364-NEXT: s_mul_i32 s10, s5, s8
+; GFX1364-NEXT: s_mul_i32 s8, s4, s8
+; GFX1364-NEXT: s_add_co_u32 s9, s9, s10
; GFX1364-NEXT: v_mov_b32_e32 v0, s8
; GFX1364-NEXT: v_mov_b32_e32 v1, s9
+; GFX1364-NEXT: s_mov_b32 s11, 0x31016000
; GFX1364-NEXT: s_mov_b32 s10, -1
; GFX1364-NEXT: s_mov_b32 s8, s2
; GFX1364-NEXT: s_mov_b32 s9, s3
@@ -7601,31 +7697,34 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1332-NEXT: s_clause 0x1
; GFX1332-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1332-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1332-NEXT: s_mov_b32 s6, exec_lo
-; GFX1332-NEXT: s_mov_b32 s7, 0
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
-; GFX1332-NEXT: s_mov_b32 s8, exec_lo
; GFX1332-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: s_bcnt1_i32_b32 s7, s6
+; GFX1332-NEXT: s_mov_b32 s6, exec_lo
; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1332-NEXT: s_cbranch_execz .LBB10_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: s_bcnt1_i32_b32 s6, s6
-; GFX1332-NEXT: s_mov_b32 s15, 0x31016000
; GFX1332-NEXT: s_wait_kmcnt 0x0
-; GFX1332-NEXT: s_mul_u64 s[6:7], s[4:5], s[6:7]
-; GFX1332-NEXT: s_mov_b32 s14, -1
-; GFX1332-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
-; GFX1332-NEXT: s_mov_b32 s12, s2
-; GFX1332-NEXT: s_mov_b32 s13, s3
+; GFX1332-NEXT: s_mul_hi_u32 s9, s4, s7
+; GFX1332-NEXT: s_mul_i32 s10, s5, s7
+; GFX1332-NEXT: s_mul_i32 s8, s4, s7
+; GFX1332-NEXT: s_add_co_u32 s9, s9, s10
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1332-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
+; GFX1332-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1332-NEXT: s_mov_b32 s10, -1
+; GFX1332-NEXT: s_mov_b32 s8, s2
+; GFX1332-NEXT: s_mov_b32 s9, s3
; GFX1332-NEXT: global_wb scope:SCOPE_DEV
; GFX1332-NEXT: s_wait_storecnt 0x0
-; GFX1332-NEXT: buffer_atomic_sub_u64 v[0:1], off, s[12:15], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1332-NEXT: buffer_atomic_sub_u64 v[0:1], off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1332-NEXT: s_wait_loadcnt 0x0
; GFX1332-NEXT: global_inv scope:SCOPE_DEV
; GFX1332-NEXT: s_wait_loadcnt 0x0
; GFX1332-NEXT: .LBB10_2:
-; GFX1332-NEXT: s_or_b32 exec_lo, exec_lo, s8
+; GFX1332-NEXT: s_or_b32 exec_lo, exec_lo, s6
; GFX1332-NEXT: s_wait_kmcnt 0x0
; GFX1332-NEXT: v_mad_co_u64_u32 v[3:4], null, s4, v2, 0
; GFX1332-NEXT: v_readfirstlane_b32 s2, v0
@@ -9385,14 +9484,15 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_and_b32 s4, s10, -4
; GFX7LESS-NEXT: s_mov_b32 s5, s11
-; GFX7LESS-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX7LESS-NEXT: s_and_b32 s0, s10, 3
-; GFX7LESS-NEXT: s_lshl_b32 s13, s0, 3
+; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0x0
; GFX7LESS-NEXT: s_and_b32 s0, s12, 0xff
+; GFX7LESS-NEXT: s_and_b32 s1, s10, 3
+; GFX7LESS-NEXT: s_lshl_b32 s13, s1, 3
+; GFX7LESS-NEXT: s_and_b32 s0, s0, 0xff
; GFX7LESS-NEXT: s_lshl_b32 s14, s0, s13
; GFX7LESS-NEXT: s_mov_b64 s[10:11], 0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s1
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s6
; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s6, -1
; GFX7LESS-NEXT: .LBB12_2: ; %atomicrmw.start
@@ -9437,14 +9537,15 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_and_b32 s4, s10, -4
; GFX8-NEXT: s_mov_b32 s5, s11
-; GFX8-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX8-NEXT: s_and_b32 s0, s10, 3
-; GFX8-NEXT: s_lshl_b32 s13, s0, 3
+; GFX8-NEXT: s_load_dword s6, s[4:5], 0x0
; GFX8-NEXT: s_and_b32 s0, s12, 0xff
+; GFX8-NEXT: s_and_b32 s1, s10, 3
+; GFX8-NEXT: s_lshl_b32 s13, s1, 3
+; GFX8-NEXT: s_and_b32 s0, s0, 0xff
; GFX8-NEXT: s_lshl_b32 s14, s0, s13
; GFX8-NEXT: s_mov_b64 s[10:11], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: v_mov_b32_e32 v1, s6
; GFX8-NEXT: s_mov_b32 s7, 0xf000
; GFX8-NEXT: s_mov_b32 s6, -1
; GFX8-NEXT: .LBB12_2: ; %atomicrmw.start
@@ -9488,14 +9589,15 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s4, s10, -4
; GFX9-NEXT: s_mov_b32 s5, s11
-; GFX9-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX9-NEXT: s_and_b32 s0, s10, 3
-; GFX9-NEXT: s_lshl_b32 s13, s0, 3
+; GFX9-NEXT: s_load_dword s6, s[4:5], 0x0
; GFX9-NEXT: s_and_b32 s0, s12, 0xff
+; GFX9-NEXT: s_and_b32 s1, s10, 3
+; GFX9-NEXT: s_lshl_b32 s13, s1, 3
+; GFX9-NEXT: s_and_b32 s0, s0, 0xff
; GFX9-NEXT: s_lshl_b32 s14, s0, s13
; GFX9-NEXT: s_mov_b64 s[10:11], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-NEXT: v_mov_b32_e32 v1, s6
; GFX9-NEXT: s_mov_b32 s7, 0xf000
; GFX9-NEXT: s_mov_b32 s6, -1
; GFX9-NEXT: .LBB12_2: ; %atomicrmw.start
@@ -9542,8 +9644,9 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1064-NEXT: s_mov_b32 s5, s11
; GFX1064-NEXT: s_and_b32 s1, s10, 3
; GFX1064-NEXT: s_load_dword s0, s[4:5], 0x0
+; GFX1064-NEXT: s_and_b32 s6, s12, 0xff
; GFX1064-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1064-NEXT: s_and_b32 s1, s12, 0xff
+; GFX1064-NEXT: s_and_b32 s1, s6, 0xff
; GFX1064-NEXT: s_mov_b64 s[10:11], 0
; GFX1064-NEXT: s_lshl_b32 s14, s1, s13
; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
@@ -9593,8 +9696,9 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1032-NEXT: s_mov_b32 s5, s11
; GFX1032-NEXT: s_and_b32 s6, s10, 3
; GFX1032-NEXT: s_load_dword s0, s[4:5], 0x0
+; GFX1032-NEXT: s_and_b32 s7, s1, 0xff
; GFX1032-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1032-NEXT: s_and_b32 s6, s1, 0xff
+; GFX1032-NEXT: s_and_b32 s6, s7, 0xff
; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
; GFX1032-NEXT: s_lshl_b32 s11, s6, s10
; GFX1032-NEXT: s_mov_b32 s6, -1
@@ -9644,8 +9748,9 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1164-TRUE16-NEXT: s_and_b32 s1, s10, 3
; GFX1164-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1164-TRUE16-NEXT: s_and_b32 s6, s12, 0xff
; GFX1164-TRUE16-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1164-TRUE16-NEXT: s_and_b32 s1, s12, 0xff
+; GFX1164-TRUE16-NEXT: s_and_b32 s1, s6, 0xff
; GFX1164-TRUE16-NEXT: s_mov_b64 s[10:11], 0
; GFX1164-TRUE16-NEXT: s_lshl_b32 s14, s1, s13
; GFX1164-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -9699,8 +9804,9 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s11
; GFX1164-FAKE16-NEXT: s_and_b32 s1, s10, 3
; GFX1164-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1164-FAKE16-NEXT: s_and_b32 s6, s12, 0xff
; GFX1164-FAKE16-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1164-FAKE16-NEXT: s_and_b32 s1, s12, 0xff
+; GFX1164-FAKE16-NEXT: s_and_b32 s1, s6, 0xff
; GFX1164-FAKE16-NEXT: s_mov_b64 s[10:11], 0
; GFX1164-FAKE16-NEXT: s_lshl_b32 s14, s1, s13
; GFX1164-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -9754,8 +9860,9 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1132-TRUE16-NEXT: s_and_b32 s6, s10, 3
; GFX1132-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1132-TRUE16-NEXT: s_and_b32 s7, s1, 0xff
; GFX1132-TRUE16-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1132-TRUE16-NEXT: s_and_b32 s6, s1, 0xff
+; GFX1132-TRUE16-NEXT: s_and_b32 s6, s7, 0xff
; GFX1132-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1132-TRUE16-NEXT: s_lshl_b32 s11, s6, s10
; GFX1132-TRUE16-NEXT: s_mov_b32 s6, -1
@@ -9807,8 +9914,9 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s11
; GFX1132-FAKE16-NEXT: s_and_b32 s6, s10, 3
; GFX1132-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1132-FAKE16-NEXT: s_and_b32 s7, s1, 0xff
; GFX1132-FAKE16-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1132-FAKE16-NEXT: s_and_b32 s6, s1, 0xff
+; GFX1132-FAKE16-NEXT: s_and_b32 s6, s7, 0xff
; GFX1132-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1132-FAKE16-NEXT: s_lshl_b32 s11, s6, s10
; GFX1132-FAKE16-NEXT: s_mov_b32 s6, -1
@@ -9860,8 +9968,9 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1264-TRUE16-NEXT: s_and_b32 s1, s10, 3
; GFX1264-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1264-TRUE16-NEXT: s_and_b32 s6, s12, 0xff
; GFX1264-TRUE16-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1264-TRUE16-NEXT: s_and_b32 s1, s12, 0xff
+; GFX1264-TRUE16-NEXT: s_and_b32 s1, s6, 0xff
; GFX1264-TRUE16-NEXT: s_mov_b64 s[10:11], 0
; GFX1264-TRUE16-NEXT: s_lshl_b32 s14, s1, s13
; GFX1264-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -9917,8 +10026,9 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s11
; GFX1264-FAKE16-NEXT: s_and_b32 s1, s10, 3
; GFX1264-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1264-FAKE16-NEXT: s_and_b32 s6, s12, 0xff
; GFX1264-FAKE16-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1264-FAKE16-NEXT: s_and_b32 s1, s12, 0xff
+; GFX1264-FAKE16-NEXT: s_and_b32 s1, s6, 0xff
; GFX1264-FAKE16-NEXT: s_mov_b64 s[10:11], 0
; GFX1264-FAKE16-NEXT: s_lshl_b32 s14, s1, s13
; GFX1264-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -9974,8 +10084,9 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1232-TRUE16-NEXT: s_and_b32 s6, s10, 3
; GFX1232-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1232-TRUE16-NEXT: s_and_b32 s7, s1, 0xff
; GFX1232-TRUE16-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1232-TRUE16-NEXT: s_and_b32 s6, s1, 0xff
+; GFX1232-TRUE16-NEXT: s_and_b32 s6, s7, 0xff
; GFX1232-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1232-TRUE16-NEXT: s_lshl_b32 s11, s6, s10
; GFX1232-TRUE16-NEXT: s_mov_b32 s6, -1
@@ -10029,8 +10140,9 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s11
; GFX1232-FAKE16-NEXT: s_and_b32 s6, s10, 3
; GFX1232-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1232-FAKE16-NEXT: s_and_b32 s7, s1, 0xff
; GFX1232-FAKE16-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1232-FAKE16-NEXT: s_and_b32 s6, s1, 0xff
+; GFX1232-FAKE16-NEXT: s_and_b32 s6, s7, 0xff
; GFX1232-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1232-FAKE16-NEXT: s_lshl_b32 s11, s6, s10
; GFX1232-FAKE16-NEXT: s_mov_b32 s6, -1
@@ -10083,8 +10195,9 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1364-TRUE16-NEXT: s_and_b64 s[4:5], s[10:11], -4
; GFX1364-TRUE16-NEXT: s_and_b32 s1, s10, 3
; GFX1364-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1364-TRUE16-NEXT: s_and_b32 s6, s12, 0xff
; GFX1364-TRUE16-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1364-TRUE16-NEXT: s_and_b32 s1, s12, 0xff
+; GFX1364-TRUE16-NEXT: s_and_b32 s1, s6, 0xff
; GFX1364-TRUE16-NEXT: s_mov_b64 s[10:11], 0
; GFX1364-TRUE16-NEXT: s_lshl_b32 s14, s1, s13
; GFX1364-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -10137,8 +10250,9 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1364-FAKE16-NEXT: s_and_b64 s[4:5], s[10:11], -4
; GFX1364-FAKE16-NEXT: s_and_b32 s1, s10, 3
; GFX1364-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1364-FAKE16-NEXT: s_and_b32 s6, s12, 0xff
; GFX1364-FAKE16-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1364-FAKE16-NEXT: s_and_b32 s1, s12, 0xff
+; GFX1364-FAKE16-NEXT: s_and_b32 s1, s6, 0xff
; GFX1364-FAKE16-NEXT: s_mov_b64 s[10:11], 0
; GFX1364-FAKE16-NEXT: s_lshl_b32 s14, s1, s13
; GFX1364-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -10191,8 +10305,9 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1332-TRUE16-NEXT: s_and_b64 s[4:5], s[10:11], -4
; GFX1332-TRUE16-NEXT: s_and_b32 s6, s10, 3
; GFX1332-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1332-TRUE16-NEXT: s_and_b32 s7, s1, 0xff
; GFX1332-TRUE16-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1332-TRUE16-NEXT: s_and_b32 s6, s1, 0xff
+; GFX1332-TRUE16-NEXT: s_and_b32 s6, s7, 0xff
; GFX1332-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1332-TRUE16-NEXT: s_lshl_b32 s11, s6, s10
; GFX1332-TRUE16-NEXT: s_mov_b32 s6, -1
@@ -10243,8 +10358,9 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1332-FAKE16-NEXT: s_and_b64 s[4:5], s[10:11], -4
; GFX1332-FAKE16-NEXT: s_and_b32 s6, s10, 3
; GFX1332-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1332-FAKE16-NEXT: s_and_b32 s7, s1, 0xff
; GFX1332-FAKE16-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1332-FAKE16-NEXT: s_and_b32 s6, s1, 0xff
+; GFX1332-FAKE16-NEXT: s_and_b32 s6, s7, 0xff
; GFX1332-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1332-FAKE16-NEXT: s_lshl_b32 s11, s6, s10
; GFX1332-FAKE16-NEXT: s_mov_b32 s6, -1
@@ -10287,17 +10403,18 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX7LESS: ; %bb.0:
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX7LESS-NEXT: s_load_dword s10, s[4:5], 0xd
-; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v4, s7, v0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v4, exec_hi, v0
+; GFX7LESS-NEXT: s_mov_b64 s[4:5], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX7LESS-NEXT: ; implicit-def: $vgpr0
; GFX7LESS-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB13_4
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_mul_i32 s6, s10, s4
+; GFX7LESS-NEXT: s_sext_i32_i8 s5, s10
+; GFX7LESS-NEXT: s_mul_i32 s6, s5, s4
; GFX7LESS-NEXT: s_and_b32 s4, s2, -4
; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: s_load_dword s7, s[4:5], 0x0
@@ -10347,17 +10464,18 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_load_dword s10, s[4:5], 0x34
-; GFX8-NEXT: s_mov_b64 s[6:7], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[4:5], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX8-NEXT: ; implicit-def: $vgpr0
; GFX8-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX8-NEXT: s_cbranch_execz .LBB13_4
; GFX8-NEXT: ; %bb.1:
-; GFX8-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mul_i32 s6, s10, s4
+; GFX8-NEXT: s_sext_i32_i8 s5, s10
+; GFX8-NEXT: s_mul_i32 s6, s5, s4
; GFX8-NEXT: s_and_b32 s4, s2, -4
; GFX8-NEXT: s_mov_b32 s5, s3
; GFX8-NEXT: s_load_dword s7, s[4:5], 0x0
@@ -10405,17 +10523,18 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_load_dword s10, s[4:5], 0x34
-; GFX9-NEXT: s_mov_b64 s[6:7], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX9-NEXT: ; implicit-def: $vgpr0
; GFX9-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX9-NEXT: s_cbranch_execz .LBB13_4
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_i32 s6, s10, s4
+; GFX9-NEXT: s_sext_i32_i8 s5, s10
+; GFX9-NEXT: s_mul_i32 s6, s5, s4
; GFX9-NEXT: s_and_b32 s4, s2, -4
; GFX9-NEXT: s_mov_b32 s5, s3
; GFX9-NEXT: s_load_dword s7, s[4:5], 0x0
@@ -10463,9 +10582,10 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1064-NEXT: s_clause 0x1
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064-NEXT: s_load_dword s10, s[4:5], 0x34
-; GFX1064-NEXT: s_mov_b64 s[6:7], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], exec
+; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1064-NEXT: ; implicit-def: $vgpr0
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX1064-NEXT: s_and_saveexec_b64 s[8:9], vcc
@@ -10476,11 +10596,11 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1064-NEXT: s_mov_b32 s5, s3
; GFX1064-NEXT: s_and_b32 s2, s2, 3
; GFX1064-NEXT: s_load_dword s3, s[4:5], 0x0
-; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1064-NEXT: s_sext_i32_i8 s7, s10
; GFX1064-NEXT: s_lshl_b32 s11, s2, 3
-; GFX1064-NEXT: s_mul_i32 s2, s10, s6
+; GFX1064-NEXT: s_mul_i32 s7, s7, s6
; GFX1064-NEXT: s_lshl_b32 s12, 0xff, s11
-; GFX1064-NEXT: s_and_b32 s2, s2, 0xff
+; GFX1064-NEXT: s_and_b32 s2, s7, 0xff
; GFX1064-NEXT: s_not_b32 s13, s12
; GFX1064-NEXT: s_lshl_b32 s14, s2, s11
; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
@@ -10520,9 +10640,10 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1032-NEXT: s_clause 0x1
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032-NEXT: s_load_dword s8, s[4:5], 0x34
-; GFX1032-NEXT: s_mov_b32 s6, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s4, exec_lo
; GFX1032-NEXT: s_mov_b32 s10, 0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s4
; GFX1032-NEXT: ; implicit-def: $vgpr0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v4
; GFX1032-NEXT: s_and_saveexec_b32 s9, vcc_lo
@@ -10533,9 +10654,9 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1032-NEXT: s_mov_b32 s5, s3
; GFX1032-NEXT: s_and_b32 s2, s2, 3
; GFX1032-NEXT: s_load_dword s7, s[4:5], 0x0
-; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1032-NEXT: s_sext_i32_i8 s11, s8
; GFX1032-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1032-NEXT: s_mul_i32 s6, s8, s6
+; GFX1032-NEXT: s_mul_i32 s6, s11, s6
; GFX1032-NEXT: s_lshl_b32 s3, 0xff, s2
; GFX1032-NEXT: s_and_b32 s6, s6, 0xff
; GFX1032-NEXT: s_not_b32 s11, s3
@@ -10576,11 +10697,12 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1164-TRUE16-NEXT: s_clause 0x1
; GFX1164-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164-TRUE16-NEXT: s_load_b32 s10, s[4:5], 0x34
-; GFX1164-TRUE16-NEXT: s_mov_b64 s[6:7], exec
+; GFX1164-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-TRUE16-NEXT: s_mov_b64 s[4:5], exec
; GFX1164-TRUE16-NEXT: s_mov_b64 s[8:9], exec
-; GFX1164-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1164-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1164-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1164-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
; GFX1164-TRUE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-TRUE16-NEXT: s_cbranch_execz .LBB13_4
@@ -10590,11 +10712,11 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1164-TRUE16-NEXT: s_and_b32 s2, s2, 3
; GFX1164-TRUE16-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1164-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1164-TRUE16-NEXT: s_sext_i32_i8 s7, s10
; GFX1164-TRUE16-NEXT: s_lshl_b32 s11, s2, 3
-; GFX1164-TRUE16-NEXT: s_mul_i32 s2, s10, s6
+; GFX1164-TRUE16-NEXT: s_mul_i32 s7, s7, s6
; GFX1164-TRUE16-NEXT: s_lshl_b32 s12, 0xff, s11
-; GFX1164-TRUE16-NEXT: s_and_b32 s2, s2, 0xff
+; GFX1164-TRUE16-NEXT: s_and_b32 s2, s7, 0xff
; GFX1164-TRUE16-NEXT: s_not_b32 s13, s12
; GFX1164-TRUE16-NEXT: s_lshl_b32 s14, s2, s11
; GFX1164-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -10639,11 +10761,12 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1164-FAKE16-NEXT: s_clause 0x1
; GFX1164-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164-FAKE16-NEXT: s_load_b32 s10, s[4:5], 0x34
-; GFX1164-FAKE16-NEXT: s_mov_b64 s[6:7], exec
+; GFX1164-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-FAKE16-NEXT: s_mov_b64 s[4:5], exec
; GFX1164-FAKE16-NEXT: s_mov_b64 s[8:9], exec
-; GFX1164-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1164-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1164-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1164-FAKE16-NEXT: ; implicit-def: $vgpr0
; GFX1164-FAKE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-FAKE16-NEXT: s_cbranch_execz .LBB13_4
@@ -10653,11 +10776,11 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1164-FAKE16-NEXT: s_and_b32 s2, s2, 3
; GFX1164-FAKE16-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1164-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1164-FAKE16-NEXT: s_sext_i32_i8 s7, s10
; GFX1164-FAKE16-NEXT: s_lshl_b32 s11, s2, 3
-; GFX1164-FAKE16-NEXT: s_mul_i32 s2, s10, s6
+; GFX1164-FAKE16-NEXT: s_mul_i32 s7, s7, s6
; GFX1164-FAKE16-NEXT: s_lshl_b32 s12, 0xff, s11
-; GFX1164-FAKE16-NEXT: s_and_b32 s2, s2, 0xff
+; GFX1164-FAKE16-NEXT: s_and_b32 s2, s7, 0xff
; GFX1164-FAKE16-NEXT: s_not_b32 s13, s12
; GFX1164-FAKE16-NEXT: s_lshl_b32 s14, s2, s11
; GFX1164-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -10702,12 +10825,12 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1132-TRUE16-NEXT: s_clause 0x1
; GFX1132-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132-TRUE16-NEXT: s_load_b32 s8, s[4:5], 0x34
-; GFX1132-TRUE16-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1132-TRUE16-NEXT: s_mov_b32 s4, exec_lo
; GFX1132-TRUE16-NEXT: s_mov_b32 s10, 0
-; GFX1132-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1132-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s4
; GFX1132-TRUE16-NEXT: s_mov_b32 s9, exec_lo
; GFX1132-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-TRUE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-TRUE16-NEXT: s_cbranch_execz .LBB13_4
; GFX1132-TRUE16-NEXT: ; %bb.1:
@@ -10716,9 +10839,9 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1132-TRUE16-NEXT: s_and_b32 s2, s2, 3
; GFX1132-TRUE16-NEXT: s_load_b32 s7, s[4:5], 0x0
-; GFX1132-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1132-TRUE16-NEXT: s_sext_i32_i8 s11, s8
; GFX1132-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1132-TRUE16-NEXT: s_mul_i32 s6, s8, s6
+; GFX1132-TRUE16-NEXT: s_mul_i32 s6, s11, s6
; GFX1132-TRUE16-NEXT: s_lshl_b32 s3, 0xff, s2
; GFX1132-TRUE16-NEXT: s_and_b32 s6, s6, 0xff
; GFX1132-TRUE16-NEXT: s_not_b32 s11, s3
@@ -10762,12 +10885,12 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1132-FAKE16-NEXT: s_clause 0x1
; GFX1132-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132-FAKE16-NEXT: s_load_b32 s8, s[4:5], 0x34
-; GFX1132-FAKE16-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1132-FAKE16-NEXT: s_mov_b32 s4, exec_lo
; GFX1132-FAKE16-NEXT: s_mov_b32 s10, 0
-; GFX1132-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1132-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s4
; GFX1132-FAKE16-NEXT: s_mov_b32 s9, exec_lo
; GFX1132-FAKE16-NEXT: ; implicit-def: $vgpr0
-; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-FAKE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-FAKE16-NEXT: s_cbranch_execz .LBB13_4
; GFX1132-FAKE16-NEXT: ; %bb.1:
@@ -10776,9 +10899,9 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1132-FAKE16-NEXT: s_and_b32 s2, s2, 3
; GFX1132-FAKE16-NEXT: s_load_b32 s7, s[4:5], 0x0
-; GFX1132-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1132-FAKE16-NEXT: s_sext_i32_i8 s11, s8
; GFX1132-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1132-FAKE16-NEXT: s_mul_i32 s6, s8, s6
+; GFX1132-FAKE16-NEXT: s_mul_i32 s6, s11, s6
; GFX1132-FAKE16-NEXT: s_lshl_b32 s3, 0xff, s2
; GFX1132-FAKE16-NEXT: s_and_b32 s6, s6, 0xff
; GFX1132-FAKE16-NEXT: s_not_b32 s11, s3
@@ -10822,11 +10945,12 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1264-TRUE16-NEXT: s_clause 0x1
; GFX1264-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1264-TRUE16-NEXT: s_load_b32 s10, s[4:5], 0x34
-; GFX1264-TRUE16-NEXT: s_mov_b64 s[6:7], exec
+; GFX1264-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1264-TRUE16-NEXT: s_mov_b64 s[4:5], exec
; GFX1264-TRUE16-NEXT: s_mov_b64 s[8:9], exec
-; GFX1264-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1264-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1264-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1264-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
; GFX1264-TRUE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1264-TRUE16-NEXT: s_cbranch_execz .LBB13_4
@@ -10836,12 +10960,11 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1264-TRUE16-NEXT: s_and_b32 s2, s2, 3
; GFX1264-TRUE16-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1264-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1264-TRUE16-NEXT: s_sext_i32_i8 s7, s10
; GFX1264-TRUE16-NEXT: s_lshl_b32 s11, s2, 3
-; GFX1264-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-TRUE16-NEXT: s_mul_i32 s2, s10, s6
+; GFX1264-TRUE16-NEXT: s_mul_i32 s7, s7, s6
; GFX1264-TRUE16-NEXT: s_lshl_b32 s12, 0xff, s11
-; GFX1264-TRUE16-NEXT: s_and_b32 s2, s2, 0xff
+; GFX1264-TRUE16-NEXT: s_and_b32 s2, s7, 0xff
; GFX1264-TRUE16-NEXT: s_not_b32 s13, s12
; GFX1264-TRUE16-NEXT: s_lshl_b32 s14, s2, s11
; GFX1264-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -10886,11 +11009,12 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1264-FAKE16-NEXT: s_clause 0x1
; GFX1264-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1264-FAKE16-NEXT: s_load_b32 s10, s[4:5], 0x34
-; GFX1264-FAKE16-NEXT: s_mov_b64 s[6:7], exec
+; GFX1264-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1264-FAKE16-NEXT: s_mov_b64 s[4:5], exec
; GFX1264-FAKE16-NEXT: s_mov_b64 s[8:9], exec
-; GFX1264-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1264-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1264-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1264-FAKE16-NEXT: ; implicit-def: $vgpr0
; GFX1264-FAKE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1264-FAKE16-NEXT: s_cbranch_execz .LBB13_4
@@ -10900,12 +11024,11 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1264-FAKE16-NEXT: s_and_b32 s2, s2, 3
; GFX1264-FAKE16-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1264-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1264-FAKE16-NEXT: s_sext_i32_i8 s7, s10
; GFX1264-FAKE16-NEXT: s_lshl_b32 s11, s2, 3
-; GFX1264-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-FAKE16-NEXT: s_mul_i32 s2, s10, s6
+; GFX1264-FAKE16-NEXT: s_mul_i32 s7, s7, s6
; GFX1264-FAKE16-NEXT: s_lshl_b32 s12, 0xff, s11
-; GFX1264-FAKE16-NEXT: s_and_b32 s2, s2, 0xff
+; GFX1264-FAKE16-NEXT: s_and_b32 s2, s7, 0xff
; GFX1264-FAKE16-NEXT: s_not_b32 s13, s12
; GFX1264-FAKE16-NEXT: s_lshl_b32 s14, s2, s11
; GFX1264-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -10950,12 +11073,12 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1232-TRUE16-NEXT: s_clause 0x1
; GFX1232-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1232-TRUE16-NEXT: s_load_b32 s8, s[4:5], 0x34
-; GFX1232-TRUE16-NEXT: s_mov_b32 s6, exec_lo
+; GFX1232-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1232-TRUE16-NEXT: s_mov_b32 s4, exec_lo
; GFX1232-TRUE16-NEXT: s_mov_b32 s10, 0
-; GFX1232-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1232-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s4
; GFX1232-TRUE16-NEXT: s_mov_b32 s9, exec_lo
; GFX1232-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1232-TRUE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1232-TRUE16-NEXT: s_cbranch_execz .LBB13_4
; GFX1232-TRUE16-NEXT: ; %bb.1:
@@ -10964,15 +11087,12 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1232-TRUE16-NEXT: s_and_b32 s2, s2, 3
; GFX1232-TRUE16-NEXT: s_load_b32 s7, s[4:5], 0x0
-; GFX1232-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1232-TRUE16-NEXT: s_sext_i32_i8 s11, s8
; GFX1232-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232-TRUE16-NEXT: s_mul_i32 s6, s8, s6
+; GFX1232-TRUE16-NEXT: s_mul_i32 s6, s11, s6
; GFX1232-TRUE16-NEXT: s_lshl_b32 s3, 0xff, s2
-; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1232-TRUE16-NEXT: s_and_b32 s6, s6, 0xff
; GFX1232-TRUE16-NEXT: s_not_b32 s11, s3
-; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1232-TRUE16-NEXT: s_lshl_b32 s12, s6, s2
; GFX1232-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x0
@@ -11014,12 +11134,12 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1232-FAKE16-NEXT: s_clause 0x1
; GFX1232-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1232-FAKE16-NEXT: s_load_b32 s8, s[4:5], 0x34
-; GFX1232-FAKE16-NEXT: s_mov_b32 s6, exec_lo
+; GFX1232-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1232-FAKE16-NEXT: s_mov_b32 s4, exec_lo
; GFX1232-FAKE16-NEXT: s_mov_b32 s10, 0
-; GFX1232-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1232-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s4
; GFX1232-FAKE16-NEXT: s_mov_b32 s9, exec_lo
; GFX1232-FAKE16-NEXT: ; implicit-def: $vgpr0
-; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1232-FAKE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1232-FAKE16-NEXT: s_cbranch_execz .LBB13_4
; GFX1232-FAKE16-NEXT: ; %bb.1:
@@ -11028,15 +11148,12 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1232-FAKE16-NEXT: s_and_b32 s2, s2, 3
; GFX1232-FAKE16-NEXT: s_load_b32 s7, s[4:5], 0x0
-; GFX1232-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1232-FAKE16-NEXT: s_sext_i32_i8 s11, s8
; GFX1232-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232-FAKE16-NEXT: s_mul_i32 s6, s8, s6
+; GFX1232-FAKE16-NEXT: s_mul_i32 s6, s11, s6
; GFX1232-FAKE16-NEXT: s_lshl_b32 s3, 0xff, s2
-; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1232-FAKE16-NEXT: s_and_b32 s6, s6, 0xff
; GFX1232-FAKE16-NEXT: s_not_b32 s11, s3
-; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1232-FAKE16-NEXT: s_lshl_b32 s12, s6, s2
; GFX1232-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x0
@@ -11078,11 +11195,12 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1364-TRUE16-NEXT: s_clause 0x1
; GFX1364-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1364-TRUE16-NEXT: s_load_b32 s10, s[4:5], 0x34 nv
-; GFX1364-TRUE16-NEXT: s_mov_b64 s[6:7], exec
+; GFX1364-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1364-TRUE16-NEXT: s_mov_b64 s[4:5], exec
; GFX1364-TRUE16-NEXT: s_mov_b64 s[8:9], exec
-; GFX1364-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1364-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1364-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1364-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
; GFX1364-TRUE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1364-TRUE16-NEXT: s_cbranch_execz .LBB13_4
@@ -11091,11 +11209,11 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1364-TRUE16-NEXT: s_and_b64 s[4:5], s[2:3], -4
; GFX1364-TRUE16-NEXT: s_and_b32 s2, s2, 3
; GFX1364-TRUE16-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1364-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1364-TRUE16-NEXT: s_sext_i32_i8 s7, s10
; GFX1364-TRUE16-NEXT: s_lshl_b32 s11, s2, 3
-; GFX1364-TRUE16-NEXT: s_mul_i32 s2, s10, s6
+; GFX1364-TRUE16-NEXT: s_mul_i32 s7, s7, s6
; GFX1364-TRUE16-NEXT: s_lshl_b32 s12, 0xff, s11
-; GFX1364-TRUE16-NEXT: s_and_b32 s2, s2, 0xff
+; GFX1364-TRUE16-NEXT: s_and_b32 s2, s7, 0xff
; GFX1364-TRUE16-NEXT: s_not_b32 s13, s12
; GFX1364-TRUE16-NEXT: s_lshl_b32 s14, s2, s11
; GFX1364-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -11139,11 +11257,12 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1364-FAKE16-NEXT: s_clause 0x1
; GFX1364-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1364-FAKE16-NEXT: s_load_b32 s10, s[4:5], 0x34 nv
-; GFX1364-FAKE16-NEXT: s_mov_b64 s[6:7], exec
+; GFX1364-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1364-FAKE16-NEXT: s_mov_b64 s[4:5], exec
; GFX1364-FAKE16-NEXT: s_mov_b64 s[8:9], exec
-; GFX1364-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1364-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX1364-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1364-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1364-FAKE16-NEXT: ; implicit-def: $vgpr0
; GFX1364-FAKE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1364-FAKE16-NEXT: s_cbranch_execz .LBB13_4
@@ -11152,11 +11271,11 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1364-FAKE16-NEXT: s_and_b64 s[4:5], s[2:3], -4
; GFX1364-FAKE16-NEXT: s_and_b32 s2, s2, 3
; GFX1364-FAKE16-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1364-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1364-FAKE16-NEXT: s_sext_i32_i8 s7, s10
; GFX1364-FAKE16-NEXT: s_lshl_b32 s11, s2, 3
-; GFX1364-FAKE16-NEXT: s_mul_i32 s2, s10, s6
+; GFX1364-FAKE16-NEXT: s_mul_i32 s7, s7, s6
; GFX1364-FAKE16-NEXT: s_lshl_b32 s12, 0xff, s11
-; GFX1364-FAKE16-NEXT: s_and_b32 s2, s2, 0xff
+; GFX1364-FAKE16-NEXT: s_and_b32 s2, s7, 0xff
; GFX1364-FAKE16-NEXT: s_not_b32 s13, s12
; GFX1364-FAKE16-NEXT: s_lshl_b32 s14, s2, s11
; GFX1364-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -11200,12 +11319,12 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1332-TRUE16-NEXT: s_clause 0x1
; GFX1332-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1332-TRUE16-NEXT: s_load_b32 s8, s[4:5], 0x34 nv
-; GFX1332-TRUE16-NEXT: s_mov_b32 s6, exec_lo
+; GFX1332-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1332-TRUE16-NEXT: s_mov_b32 s4, exec_lo
; GFX1332-TRUE16-NEXT: s_mov_b32 s10, 0
-; GFX1332-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1332-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s4
; GFX1332-TRUE16-NEXT: s_mov_b32 s9, exec_lo
; GFX1332-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1332-TRUE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1332-TRUE16-NEXT: s_cbranch_execz .LBB13_4
; GFX1332-TRUE16-NEXT: ; %bb.1:
@@ -11213,9 +11332,9 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1332-TRUE16-NEXT: s_and_b64 s[4:5], s[2:3], -4
; GFX1332-TRUE16-NEXT: s_and_b32 s2, s2, 3
; GFX1332-TRUE16-NEXT: s_load_b32 s7, s[4:5], 0x0
-; GFX1332-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1332-TRUE16-NEXT: s_sext_i32_i8 s11, s8
; GFX1332-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1332-TRUE16-NEXT: s_mul_i32 s6, s8, s6
+; GFX1332-TRUE16-NEXT: s_mul_i32 s6, s11, s6
; GFX1332-TRUE16-NEXT: s_lshl_b32 s3, 0xff, s2
; GFX1332-TRUE16-NEXT: s_and_b32 s6, s6, 0xff
; GFX1332-TRUE16-NEXT: s_not_b32 s11, s3
@@ -11259,12 +11378,12 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1332-FAKE16-NEXT: s_clause 0x1
; GFX1332-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1332-FAKE16-NEXT: s_load_b32 s8, s[4:5], 0x34 nv
-; GFX1332-FAKE16-NEXT: s_mov_b32 s6, exec_lo
+; GFX1332-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1332-FAKE16-NEXT: s_mov_b32 s4, exec_lo
; GFX1332-FAKE16-NEXT: s_mov_b32 s10, 0
-; GFX1332-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1332-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s4
; GFX1332-FAKE16-NEXT: s_mov_b32 s9, exec_lo
; GFX1332-FAKE16-NEXT: ; implicit-def: $vgpr0
-; GFX1332-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1332-FAKE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1332-FAKE16-NEXT: s_cbranch_execz .LBB13_4
; GFX1332-FAKE16-NEXT: ; %bb.1:
@@ -11272,9 +11391,9 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1332-FAKE16-NEXT: s_and_b64 s[4:5], s[2:3], -4
; GFX1332-FAKE16-NEXT: s_and_b32 s2, s2, 3
; GFX1332-FAKE16-NEXT: s_load_b32 s7, s[4:5], 0x0
-; GFX1332-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1332-FAKE16-NEXT: s_sext_i32_i8 s11, s8
; GFX1332-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1332-FAKE16-NEXT: s_mul_i32 s6, s8, s6
+; GFX1332-FAKE16-NEXT: s_mul_i32 s6, s11, s6
; GFX1332-FAKE16-NEXT: s_lshl_b32 s3, 0xff, s2
; GFX1332-FAKE16-NEXT: s_and_b32 s6, s6, 0xff
; GFX1332-FAKE16-NEXT: s_not_b32 s11, s3
@@ -11786,9 +11905,10 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX7LESS-NEXT: s_and_b32 s4, s10, -4
; GFX7LESS-NEXT: s_mov_b32 s5, s11
; GFX7LESS-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX7LESS-NEXT: s_and_b32 s0, s10, 3
-; GFX7LESS-NEXT: s_lshl_b32 s13, s0, 3
; GFX7LESS-NEXT: s_and_b32 s0, s12, 0xffff
+; GFX7LESS-NEXT: s_and_b32 s6, s10, 3
+; GFX7LESS-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX7LESS-NEXT: s_lshl_b32 s13, s6, 3
; GFX7LESS-NEXT: s_lshl_b32 s14, s0, s13
; GFX7LESS-NEXT: s_mov_b64 s[10:11], 0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
@@ -11837,14 +11957,15 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_and_b32 s4, s10, -4
; GFX8-NEXT: s_mov_b32 s5, s11
-; GFX8-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX8-NEXT: s_and_b32 s0, s10, 3
-; GFX8-NEXT: s_lshl_b32 s13, s0, 3
-; GFX8-NEXT: s_and_b32 s0, 0xffff, s12
+; GFX8-NEXT: s_load_dword s6, s[4:5], 0x0
+; GFX8-NEXT: s_and_b32 s0, s12, 0xffff
+; GFX8-NEXT: s_and_b32 s1, s10, 3
+; GFX8-NEXT: s_lshl_b32 s13, s1, 3
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
; GFX8-NEXT: s_lshl_b32 s14, s0, s13
; GFX8-NEXT: s_mov_b64 s[10:11], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: v_mov_b32_e32 v1, s6
; GFX8-NEXT: s_mov_b32 s7, 0xf000
; GFX8-NEXT: s_mov_b32 s6, -1
; GFX8-NEXT: .LBB15_2: ; %atomicrmw.start
@@ -11888,14 +12009,15 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s4, s10, -4
; GFX9-NEXT: s_mov_b32 s5, s11
-; GFX9-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX9-NEXT: s_and_b32 s0, s10, 3
-; GFX9-NEXT: s_lshl_b32 s13, s0, 3
-; GFX9-NEXT: s_and_b32 s0, 0xffff, s12
+; GFX9-NEXT: s_load_dword s6, s[4:5], 0x0
+; GFX9-NEXT: s_and_b32 s0, s12, 0xffff
+; GFX9-NEXT: s_and_b32 s1, s10, 3
+; GFX9-NEXT: s_lshl_b32 s13, s1, 3
+; GFX9-NEXT: s_and_b32 s0, 0xffff, s0
; GFX9-NEXT: s_lshl_b32 s14, s0, s13
; GFX9-NEXT: s_mov_b64 s[10:11], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-NEXT: v_mov_b32_e32 v1, s6
; GFX9-NEXT: s_mov_b32 s7, 0xf000
; GFX9-NEXT: s_mov_b32 s6, -1
; GFX9-NEXT: .LBB15_2: ; %atomicrmw.start
@@ -11942,8 +12064,9 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1064-NEXT: s_mov_b32 s5, s11
; GFX1064-NEXT: s_and_b32 s1, s10, 3
; GFX1064-NEXT: s_load_dword s0, s[4:5], 0x0
+; GFX1064-NEXT: s_and_b32 s6, s12, 0xffff
; GFX1064-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1064-NEXT: s_and_b32 s1, 0xffff, s12
+; GFX1064-NEXT: s_and_b32 s1, 0xffff, s6
; GFX1064-NEXT: s_mov_b64 s[10:11], 0
; GFX1064-NEXT: s_lshl_b32 s14, s1, s13
; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
@@ -11993,8 +12116,9 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1032-NEXT: s_mov_b32 s5, s11
; GFX1032-NEXT: s_and_b32 s6, s10, 3
; GFX1032-NEXT: s_load_dword s0, s[4:5], 0x0
+; GFX1032-NEXT: s_and_b32 s7, s1, 0xffff
; GFX1032-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1032-NEXT: s_and_b32 s6, 0xffff, s1
+; GFX1032-NEXT: s_and_b32 s6, 0xffff, s7
; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
; GFX1032-NEXT: s_lshl_b32 s11, s6, s10
; GFX1032-NEXT: s_mov_b32 s6, -1
@@ -12044,8 +12168,9 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1164-TRUE16-NEXT: s_and_b32 s1, s10, 3
; GFX1164-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1164-TRUE16-NEXT: s_and_b32 s6, s12, 0xffff
; GFX1164-TRUE16-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1164-TRUE16-NEXT: s_and_b32 s1, 0xffff, s12
+; GFX1164-TRUE16-NEXT: s_and_b32 s1, 0xffff, s6
; GFX1164-TRUE16-NEXT: s_mov_b64 s[10:11], 0
; GFX1164-TRUE16-NEXT: s_lshl_b32 s14, s1, s13
; GFX1164-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -12099,8 +12224,9 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s11
; GFX1164-FAKE16-NEXT: s_and_b32 s1, s10, 3
; GFX1164-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1164-FAKE16-NEXT: s_and_b32 s6, s12, 0xffff
; GFX1164-FAKE16-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1164-FAKE16-NEXT: s_and_b32 s1, 0xffff, s12
+; GFX1164-FAKE16-NEXT: s_and_b32 s1, 0xffff, s6
; GFX1164-FAKE16-NEXT: s_mov_b64 s[10:11], 0
; GFX1164-FAKE16-NEXT: s_lshl_b32 s14, s1, s13
; GFX1164-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -12154,8 +12280,9 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1132-TRUE16-NEXT: s_and_b32 s6, s10, 3
; GFX1132-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1132-TRUE16-NEXT: s_and_b32 s7, s1, 0xffff
; GFX1132-TRUE16-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1132-TRUE16-NEXT: s_and_b32 s6, 0xffff, s1
+; GFX1132-TRUE16-NEXT: s_and_b32 s6, 0xffff, s7
; GFX1132-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1132-TRUE16-NEXT: s_lshl_b32 s11, s6, s10
; GFX1132-TRUE16-NEXT: s_mov_b32 s6, -1
@@ -12207,8 +12334,9 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s11
; GFX1132-FAKE16-NEXT: s_and_b32 s6, s10, 3
; GFX1132-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1132-FAKE16-NEXT: s_and_b32 s7, s1, 0xffff
; GFX1132-FAKE16-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1132-FAKE16-NEXT: s_and_b32 s6, 0xffff, s1
+; GFX1132-FAKE16-NEXT: s_and_b32 s6, 0xffff, s7
; GFX1132-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1132-FAKE16-NEXT: s_lshl_b32 s11, s6, s10
; GFX1132-FAKE16-NEXT: s_mov_b32 s6, -1
@@ -12260,8 +12388,9 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1264-TRUE16-NEXT: s_and_b32 s1, s10, 3
; GFX1264-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1264-TRUE16-NEXT: s_and_b32 s6, s12, 0xffff
; GFX1264-TRUE16-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1264-TRUE16-NEXT: s_and_b32 s1, 0xffff, s12
+; GFX1264-TRUE16-NEXT: s_and_b32 s1, 0xffff, s6
; GFX1264-TRUE16-NEXT: s_mov_b64 s[10:11], 0
; GFX1264-TRUE16-NEXT: s_lshl_b32 s14, s1, s13
; GFX1264-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -12317,8 +12446,9 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s11
; GFX1264-FAKE16-NEXT: s_and_b32 s1, s10, 3
; GFX1264-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1264-FAKE16-NEXT: s_and_b32 s6, s12, 0xffff
; GFX1264-FAKE16-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1264-FAKE16-NEXT: s_and_b32 s1, 0xffff, s12
+; GFX1264-FAKE16-NEXT: s_and_b32 s1, 0xffff, s6
; GFX1264-FAKE16-NEXT: s_mov_b64 s[10:11], 0
; GFX1264-FAKE16-NEXT: s_lshl_b32 s14, s1, s13
; GFX1264-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -12374,8 +12504,9 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1232-TRUE16-NEXT: s_and_b32 s6, s10, 3
; GFX1232-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1232-TRUE16-NEXT: s_and_b32 s7, s1, 0xffff
; GFX1232-TRUE16-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1232-TRUE16-NEXT: s_and_b32 s6, 0xffff, s1
+; GFX1232-TRUE16-NEXT: s_and_b32 s6, 0xffff, s7
; GFX1232-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1232-TRUE16-NEXT: s_lshl_b32 s11, s6, s10
; GFX1232-TRUE16-NEXT: s_mov_b32 s6, -1
@@ -12429,8 +12560,9 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s11
; GFX1232-FAKE16-NEXT: s_and_b32 s6, s10, 3
; GFX1232-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1232-FAKE16-NEXT: s_and_b32 s7, s1, 0xffff
; GFX1232-FAKE16-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1232-FAKE16-NEXT: s_and_b32 s6, 0xffff, s1
+; GFX1232-FAKE16-NEXT: s_and_b32 s6, 0xffff, s7
; GFX1232-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1232-FAKE16-NEXT: s_lshl_b32 s11, s6, s10
; GFX1232-FAKE16-NEXT: s_mov_b32 s6, -1
@@ -12483,8 +12615,9 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1364-TRUE16-NEXT: s_and_b64 s[4:5], s[10:11], -4
; GFX1364-TRUE16-NEXT: s_and_b32 s1, s10, 3
; GFX1364-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1364-TRUE16-NEXT: s_and_b32 s6, s12, 0xffff
; GFX1364-TRUE16-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1364-TRUE16-NEXT: s_and_b32 s1, 0xffff, s12
+; GFX1364-TRUE16-NEXT: s_and_b32 s1, 0xffff, s6
; GFX1364-TRUE16-NEXT: s_mov_b64 s[10:11], 0
; GFX1364-TRUE16-NEXT: s_lshl_b32 s14, s1, s13
; GFX1364-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -12537,8 +12670,9 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1364-FAKE16-NEXT: s_and_b64 s[4:5], s[10:11], -4
; GFX1364-FAKE16-NEXT: s_and_b32 s1, s10, 3
; GFX1364-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1364-FAKE16-NEXT: s_and_b32 s6, s12, 0xffff
; GFX1364-FAKE16-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1364-FAKE16-NEXT: s_and_b32 s1, 0xffff, s12
+; GFX1364-FAKE16-NEXT: s_and_b32 s1, 0xffff, s6
; GFX1364-FAKE16-NEXT: s_mov_b64 s[10:11], 0
; GFX1364-FAKE16-NEXT: s_lshl_b32 s14, s1, s13
; GFX1364-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -12591,8 +12725,9 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1332-TRUE16-NEXT: s_and_b64 s[4:5], s[10:11], -4
; GFX1332-TRUE16-NEXT: s_and_b32 s6, s10, 3
; GFX1332-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1332-TRUE16-NEXT: s_and_b32 s7, s1, 0xffff
; GFX1332-TRUE16-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1332-TRUE16-NEXT: s_and_b32 s6, 0xffff, s1
+; GFX1332-TRUE16-NEXT: s_and_b32 s6, 0xffff, s7
; GFX1332-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1332-TRUE16-NEXT: s_lshl_b32 s11, s6, s10
; GFX1332-TRUE16-NEXT: s_mov_b32 s6, -1
@@ -12643,8 +12778,9 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1332-FAKE16-NEXT: s_and_b64 s[4:5], s[10:11], -4
; GFX1332-FAKE16-NEXT: s_and_b32 s6, s10, 3
; GFX1332-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1332-FAKE16-NEXT: s_and_b32 s7, s1, 0xffff
; GFX1332-FAKE16-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1332-FAKE16-NEXT: s_and_b32 s6, 0xffff, s1
+; GFX1332-FAKE16-NEXT: s_and_b32 s6, 0xffff, s7
; GFX1332-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1332-FAKE16-NEXT: s_lshl_b32 s11, s6, s10
; GFX1332-FAKE16-NEXT: s_mov_b32 s6, -1
@@ -12687,26 +12823,27 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX7LESS: ; %bb.0:
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX7LESS-NEXT: s_load_dword s10, s[4:5], 0xd
-; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v4, s7, v0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v4, exec_hi, v0
+; GFX7LESS-NEXT: s_mov_b64 s[4:5], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX7LESS-NEXT: ; implicit-def: $vgpr0
; GFX7LESS-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB16_4
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_mul_i32 s6, s10, s4
+; GFX7LESS-NEXT: s_sext_i32_i16 s5, s10
+; GFX7LESS-NEXT: s_mul_i32 s5, s5, s4
+; GFX7LESS-NEXT: s_and_b32 s6, s5, 0xffff
; GFX7LESS-NEXT: s_and_b32 s4, s2, -4
; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: s_load_dword s7, s[4:5], 0x0
; GFX7LESS-NEXT: s_and_b32 s2, s2, 3
; GFX7LESS-NEXT: s_lshl_b32 s11, s2, 3
; GFX7LESS-NEXT: s_lshl_b32 s12, 0xffff, s11
-; GFX7LESS-NEXT: s_and_b32 s2, s6, 0xffff
; GFX7LESS-NEXT: s_not_b32 s13, s12
-; GFX7LESS-NEXT: s_lshl_b32 s14, s2, s11
+; GFX7LESS-NEXT: s_lshl_b32 s14, s6, s11
; GFX7LESS-NEXT: s_mov_b64 s[2:3], 0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v1, s7
@@ -12747,24 +12884,25 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_load_dword s10, s[4:5], 0x34
-; GFX8-NEXT: s_mov_b64 s[6:7], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[4:5], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX8-NEXT: ; implicit-def: $vgpr0
; GFX8-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX8-NEXT: s_cbranch_execz .LBB16_4
; GFX8-NEXT: ; %bb.1:
-; GFX8-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mul_i32 s6, s10, s4
+; GFX8-NEXT: s_sext_i32_i16 s5, s10
+; GFX8-NEXT: s_mul_i32 s6, s5, s4
; GFX8-NEXT: s_and_b32 s4, s2, -4
; GFX8-NEXT: s_mov_b32 s5, s3
; GFX8-NEXT: s_load_dword s7, s[4:5], 0x0
; GFX8-NEXT: s_and_b32 s2, s2, 3
; GFX8-NEXT: s_lshl_b32 s11, s2, 3
; GFX8-NEXT: s_lshl_b32 s12, 0xffff, s11
-; GFX8-NEXT: s_and_b32 s2, s6, 0xffff
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s6
; GFX8-NEXT: s_not_b32 s13, s12
; GFX8-NEXT: s_lshl_b32 s14, s2, s11
; GFX8-NEXT: s_mov_b64 s[2:3], 0
@@ -12805,24 +12943,25 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_load_dword s10, s[4:5], 0x34
-; GFX9-NEXT: s_mov_b64 s[6:7], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX9-NEXT: ; implicit-def: $vgpr0
; GFX9-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX9-NEXT: s_cbranch_execz .LBB16_4
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_i32 s6, s10, s4
+; GFX9-NEXT: s_sext_i32_i16 s5, s10
+; GFX9-NEXT: s_mul_i32 s6, s5, s4
; GFX9-NEXT: s_and_b32 s4, s2, -4
; GFX9-NEXT: s_mov_b32 s5, s3
; GFX9-NEXT: s_load_dword s7, s[4:5], 0x0
; GFX9-NEXT: s_and_b32 s2, s2, 3
; GFX9-NEXT: s_lshl_b32 s11, s2, 3
; GFX9-NEXT: s_lshl_b32 s12, 0xffff, s11
-; GFX9-NEXT: s_and_b32 s2, s6, 0xffff
+; GFX9-NEXT: s_and_b32 s2, 0xffff, s6
; GFX9-NEXT: s_not_b32 s13, s12
; GFX9-NEXT: s_lshl_b32 s14, s2, s11
; GFX9-NEXT: s_mov_b64 s[2:3], 0
@@ -12863,9 +13002,10 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1064-NEXT: s_clause 0x1
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064-NEXT: s_load_dword s10, s[4:5], 0x34
-; GFX1064-NEXT: s_mov_b64 s[6:7], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], exec
+; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1064-NEXT: ; implicit-def: $vgpr0
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX1064-NEXT: s_and_saveexec_b64 s[8:9], vcc
@@ -12876,11 +13016,11 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1064-NEXT: s_mov_b32 s5, s3
; GFX1064-NEXT: s_and_b32 s2, s2, 3
; GFX1064-NEXT: s_load_dword s3, s[4:5], 0x0
-; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1064-NEXT: s_sext_i32_i16 s7, s10
; GFX1064-NEXT: s_lshl_b32 s11, s2, 3
-; GFX1064-NEXT: s_mul_i32 s2, s10, s6
+; GFX1064-NEXT: s_mul_i32 s7, s7, s6
; GFX1064-NEXT: s_lshl_b32 s12, 0xffff, s11
-; GFX1064-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX1064-NEXT: s_and_b32 s2, 0xffff, s7
; GFX1064-NEXT: s_not_b32 s13, s12
; GFX1064-NEXT: s_lshl_b32 s14, s2, s11
; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
@@ -12920,9 +13060,10 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1032-NEXT: s_clause 0x1
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032-NEXT: s_load_dword s8, s[4:5], 0x34
-; GFX1032-NEXT: s_mov_b32 s6, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s4, exec_lo
; GFX1032-NEXT: s_mov_b32 s10, 0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s4
; GFX1032-NEXT: ; implicit-def: $vgpr0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v4
; GFX1032-NEXT: s_and_saveexec_b32 s9, vcc_lo
@@ -12933,11 +13074,11 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1032-NEXT: s_mov_b32 s5, s3
; GFX1032-NEXT: s_and_b32 s2, s2, 3
; GFX1032-NEXT: s_load_dword s7, s[4:5], 0x0
-; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1032-NEXT: s_sext_i32_i16 s11, s8
; GFX1032-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1032-NEXT: s_mul_i32 s6, s8, s6
+; GFX1032-NEXT: s_mul_i32 s6, s11, s6
; GFX1032-NEXT: s_lshl_b32 s3, 0xffff, s2
-; GFX1032-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1032-NEXT: s_and_b32 s6, 0xffff, s6
; GFX1032-NEXT: s_not_b32 s11, s3
; GFX1032-NEXT: s_lshl_b32 s12, s6, s2
; GFX1032-NEXT: s_mov_b32 s6, -1
@@ -12976,11 +13117,12 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1164-TRUE16-NEXT: s_clause 0x1
; GFX1164-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164-TRUE16-NEXT: s_load_b32 s10, s[4:5], 0x34
-; GFX1164-TRUE16-NEXT: s_mov_b64 s[6:7], exec
+; GFX1164-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-TRUE16-NEXT: s_mov_b64 s[4:5], exec
; GFX1164-TRUE16-NEXT: s_mov_b64 s[8:9], exec
-; GFX1164-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1164-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1164-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1164-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
; GFX1164-TRUE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-TRUE16-NEXT: s_cbranch_execz .LBB16_4
@@ -12990,11 +13132,11 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1164-TRUE16-NEXT: s_and_b32 s2, s2, 3
; GFX1164-TRUE16-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1164-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1164-TRUE16-NEXT: s_sext_i32_i16 s7, s10
; GFX1164-TRUE16-NEXT: s_lshl_b32 s11, s2, 3
-; GFX1164-TRUE16-NEXT: s_mul_i32 s2, s10, s6
+; GFX1164-TRUE16-NEXT: s_mul_i32 s7, s7, s6
; GFX1164-TRUE16-NEXT: s_lshl_b32 s12, 0xffff, s11
-; GFX1164-TRUE16-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX1164-TRUE16-NEXT: s_and_b32 s2, 0xffff, s7
; GFX1164-TRUE16-NEXT: s_not_b32 s13, s12
; GFX1164-TRUE16-NEXT: s_lshl_b32 s14, s2, s11
; GFX1164-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -13039,11 +13181,12 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1164-FAKE16-NEXT: s_clause 0x1
; GFX1164-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164-FAKE16-NEXT: s_load_b32 s10, s[4:5], 0x34
-; GFX1164-FAKE16-NEXT: s_mov_b64 s[6:7], exec
+; GFX1164-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-FAKE16-NEXT: s_mov_b64 s[4:5], exec
; GFX1164-FAKE16-NEXT: s_mov_b64 s[8:9], exec
-; GFX1164-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1164-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1164-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1164-FAKE16-NEXT: ; implicit-def: $vgpr0
; GFX1164-FAKE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-FAKE16-NEXT: s_cbranch_execz .LBB16_4
@@ -13053,11 +13196,11 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1164-FAKE16-NEXT: s_and_b32 s2, s2, 3
; GFX1164-FAKE16-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1164-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1164-FAKE16-NEXT: s_sext_i32_i16 s7, s10
; GFX1164-FAKE16-NEXT: s_lshl_b32 s11, s2, 3
-; GFX1164-FAKE16-NEXT: s_mul_i32 s2, s10, s6
+; GFX1164-FAKE16-NEXT: s_mul_i32 s7, s7, s6
; GFX1164-FAKE16-NEXT: s_lshl_b32 s12, 0xffff, s11
-; GFX1164-FAKE16-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX1164-FAKE16-NEXT: s_and_b32 s2, 0xffff, s7
; GFX1164-FAKE16-NEXT: s_not_b32 s13, s12
; GFX1164-FAKE16-NEXT: s_lshl_b32 s14, s2, s11
; GFX1164-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -13102,12 +13245,12 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1132-TRUE16-NEXT: s_clause 0x1
; GFX1132-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132-TRUE16-NEXT: s_load_b32 s8, s[4:5], 0x34
-; GFX1132-TRUE16-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1132-TRUE16-NEXT: s_mov_b32 s4, exec_lo
; GFX1132-TRUE16-NEXT: s_mov_b32 s10, 0
-; GFX1132-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1132-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s4
; GFX1132-TRUE16-NEXT: s_mov_b32 s9, exec_lo
; GFX1132-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-TRUE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-TRUE16-NEXT: s_cbranch_execz .LBB16_4
; GFX1132-TRUE16-NEXT: ; %bb.1:
@@ -13116,11 +13259,11 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1132-TRUE16-NEXT: s_and_b32 s2, s2, 3
; GFX1132-TRUE16-NEXT: s_load_b32 s7, s[4:5], 0x0
-; GFX1132-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1132-TRUE16-NEXT: s_sext_i32_i16 s11, s8
; GFX1132-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1132-TRUE16-NEXT: s_mul_i32 s6, s8, s6
+; GFX1132-TRUE16-NEXT: s_mul_i32 s6, s11, s6
; GFX1132-TRUE16-NEXT: s_lshl_b32 s3, 0xffff, s2
-; GFX1132-TRUE16-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1132-TRUE16-NEXT: s_and_b32 s6, 0xffff, s6
; GFX1132-TRUE16-NEXT: s_not_b32 s11, s3
; GFX1132-TRUE16-NEXT: s_lshl_b32 s12, s6, s2
; GFX1132-TRUE16-NEXT: s_mov_b32 s6, -1
@@ -13162,12 +13305,12 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1132-FAKE16-NEXT: s_clause 0x1
; GFX1132-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132-FAKE16-NEXT: s_load_b32 s8, s[4:5], 0x34
-; GFX1132-FAKE16-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1132-FAKE16-NEXT: s_mov_b32 s4, exec_lo
; GFX1132-FAKE16-NEXT: s_mov_b32 s10, 0
-; GFX1132-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1132-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s4
; GFX1132-FAKE16-NEXT: s_mov_b32 s9, exec_lo
; GFX1132-FAKE16-NEXT: ; implicit-def: $vgpr0
-; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-FAKE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-FAKE16-NEXT: s_cbranch_execz .LBB16_4
; GFX1132-FAKE16-NEXT: ; %bb.1:
@@ -13176,11 +13319,11 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1132-FAKE16-NEXT: s_and_b32 s2, s2, 3
; GFX1132-FAKE16-NEXT: s_load_b32 s7, s[4:5], 0x0
-; GFX1132-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1132-FAKE16-NEXT: s_sext_i32_i16 s11, s8
; GFX1132-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1132-FAKE16-NEXT: s_mul_i32 s6, s8, s6
+; GFX1132-FAKE16-NEXT: s_mul_i32 s6, s11, s6
; GFX1132-FAKE16-NEXT: s_lshl_b32 s3, 0xffff, s2
-; GFX1132-FAKE16-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1132-FAKE16-NEXT: s_and_b32 s6, 0xffff, s6
; GFX1132-FAKE16-NEXT: s_not_b32 s11, s3
; GFX1132-FAKE16-NEXT: s_lshl_b32 s12, s6, s2
; GFX1132-FAKE16-NEXT: s_mov_b32 s6, -1
@@ -13222,11 +13365,12 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1264-TRUE16-NEXT: s_clause 0x1
; GFX1264-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1264-TRUE16-NEXT: s_load_b32 s10, s[4:5], 0x34
-; GFX1264-TRUE16-NEXT: s_mov_b64 s[6:7], exec
+; GFX1264-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1264-TRUE16-NEXT: s_mov_b64 s[4:5], exec
; GFX1264-TRUE16-NEXT: s_mov_b64 s[8:9], exec
-; GFX1264-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1264-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1264-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1264-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
; GFX1264-TRUE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1264-TRUE16-NEXT: s_cbranch_execz .LBB16_4
@@ -13236,12 +13380,11 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1264-TRUE16-NEXT: s_and_b32 s2, s2, 3
; GFX1264-TRUE16-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1264-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1264-TRUE16-NEXT: s_sext_i32_i16 s7, s10
; GFX1264-TRUE16-NEXT: s_lshl_b32 s11, s2, 3
-; GFX1264-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-TRUE16-NEXT: s_mul_i32 s2, s10, s6
+; GFX1264-TRUE16-NEXT: s_mul_i32 s7, s7, s6
; GFX1264-TRUE16-NEXT: s_lshl_b32 s12, 0xffff, s11
-; GFX1264-TRUE16-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX1264-TRUE16-NEXT: s_and_b32 s2, 0xffff, s7
; GFX1264-TRUE16-NEXT: s_not_b32 s13, s12
; GFX1264-TRUE16-NEXT: s_lshl_b32 s14, s2, s11
; GFX1264-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -13286,11 +13429,12 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1264-FAKE16-NEXT: s_clause 0x1
; GFX1264-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1264-FAKE16-NEXT: s_load_b32 s10, s[4:5], 0x34
-; GFX1264-FAKE16-NEXT: s_mov_b64 s[6:7], exec
+; GFX1264-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1264-FAKE16-NEXT: s_mov_b64 s[4:5], exec
; GFX1264-FAKE16-NEXT: s_mov_b64 s[8:9], exec
-; GFX1264-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1264-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1264-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1264-FAKE16-NEXT: ; implicit-def: $vgpr0
; GFX1264-FAKE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1264-FAKE16-NEXT: s_cbranch_execz .LBB16_4
@@ -13300,12 +13444,11 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1264-FAKE16-NEXT: s_and_b32 s2, s2, 3
; GFX1264-FAKE16-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1264-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1264-FAKE16-NEXT: s_sext_i32_i16 s7, s10
; GFX1264-FAKE16-NEXT: s_lshl_b32 s11, s2, 3
-; GFX1264-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-FAKE16-NEXT: s_mul_i32 s2, s10, s6
+; GFX1264-FAKE16-NEXT: s_mul_i32 s7, s7, s6
; GFX1264-FAKE16-NEXT: s_lshl_b32 s12, 0xffff, s11
-; GFX1264-FAKE16-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX1264-FAKE16-NEXT: s_and_b32 s2, 0xffff, s7
; GFX1264-FAKE16-NEXT: s_not_b32 s13, s12
; GFX1264-FAKE16-NEXT: s_lshl_b32 s14, s2, s11
; GFX1264-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -13350,12 +13493,12 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1232-TRUE16-NEXT: s_clause 0x1
; GFX1232-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1232-TRUE16-NEXT: s_load_b32 s8, s[4:5], 0x34
-; GFX1232-TRUE16-NEXT: s_mov_b32 s6, exec_lo
+; GFX1232-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1232-TRUE16-NEXT: s_mov_b32 s4, exec_lo
; GFX1232-TRUE16-NEXT: s_mov_b32 s10, 0
-; GFX1232-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1232-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s4
; GFX1232-TRUE16-NEXT: s_mov_b32 s9, exec_lo
; GFX1232-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1232-TRUE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1232-TRUE16-NEXT: s_cbranch_execz .LBB16_4
; GFX1232-TRUE16-NEXT: ; %bb.1:
@@ -13364,15 +13507,12 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1232-TRUE16-NEXT: s_and_b32 s2, s2, 3
; GFX1232-TRUE16-NEXT: s_load_b32 s7, s[4:5], 0x0
-; GFX1232-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1232-TRUE16-NEXT: s_sext_i32_i16 s11, s8
; GFX1232-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232-TRUE16-NEXT: s_mul_i32 s6, s8, s6
+; GFX1232-TRUE16-NEXT: s_mul_i32 s6, s11, s6
; GFX1232-TRUE16-NEXT: s_lshl_b32 s3, 0xffff, s2
-; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232-TRUE16-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1232-TRUE16-NEXT: s_and_b32 s6, 0xffff, s6
; GFX1232-TRUE16-NEXT: s_not_b32 s11, s3
-; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1232-TRUE16-NEXT: s_lshl_b32 s12, s6, s2
; GFX1232-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x0
@@ -13414,12 +13554,12 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1232-FAKE16-NEXT: s_clause 0x1
; GFX1232-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1232-FAKE16-NEXT: s_load_b32 s8, s[4:5], 0x34
-; GFX1232-FAKE16-NEXT: s_mov_b32 s6, exec_lo
+; GFX1232-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1232-FAKE16-NEXT: s_mov_b32 s4, exec_lo
; GFX1232-FAKE16-NEXT: s_mov_b32 s10, 0
-; GFX1232-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1232-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s4
; GFX1232-FAKE16-NEXT: s_mov_b32 s9, exec_lo
; GFX1232-FAKE16-NEXT: ; implicit-def: $vgpr0
-; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1232-FAKE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1232-FAKE16-NEXT: s_cbranch_execz .LBB16_4
; GFX1232-FAKE16-NEXT: ; %bb.1:
@@ -13428,15 +13568,12 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1232-FAKE16-NEXT: s_and_b32 s2, s2, 3
; GFX1232-FAKE16-NEXT: s_load_b32 s7, s[4:5], 0x0
-; GFX1232-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1232-FAKE16-NEXT: s_sext_i32_i16 s11, s8
; GFX1232-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232-FAKE16-NEXT: s_mul_i32 s6, s8, s6
+; GFX1232-FAKE16-NEXT: s_mul_i32 s6, s11, s6
; GFX1232-FAKE16-NEXT: s_lshl_b32 s3, 0xffff, s2
-; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232-FAKE16-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1232-FAKE16-NEXT: s_and_b32 s6, 0xffff, s6
; GFX1232-FAKE16-NEXT: s_not_b32 s11, s3
-; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1232-FAKE16-NEXT: s_lshl_b32 s12, s6, s2
; GFX1232-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x0
@@ -13478,11 +13615,12 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1364-TRUE16-NEXT: s_clause 0x1
; GFX1364-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1364-TRUE16-NEXT: s_load_b32 s10, s[4:5], 0x34 nv
-; GFX1364-TRUE16-NEXT: s_mov_b64 s[6:7], exec
+; GFX1364-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1364-TRUE16-NEXT: s_mov_b64 s[4:5], exec
; GFX1364-TRUE16-NEXT: s_mov_b64 s[8:9], exec
-; GFX1364-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1364-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1364-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1364-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
; GFX1364-TRUE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1364-TRUE16-NEXT: s_cbranch_execz .LBB16_4
@@ -13491,11 +13629,11 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1364-TRUE16-NEXT: s_and_b64 s[4:5], s[2:3], -4
; GFX1364-TRUE16-NEXT: s_and_b32 s2, s2, 3
; GFX1364-TRUE16-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1364-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1364-TRUE16-NEXT: s_sext_i32_i16 s7, s10
; GFX1364-TRUE16-NEXT: s_lshl_b32 s11, s2, 3
-; GFX1364-TRUE16-NEXT: s_mul_i32 s2, s10, s6
+; GFX1364-TRUE16-NEXT: s_mul_i32 s7, s7, s6
; GFX1364-TRUE16-NEXT: s_lshl_b32 s12, 0xffff, s11
-; GFX1364-TRUE16-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX1364-TRUE16-NEXT: s_and_b32 s2, 0xffff, s7
; GFX1364-TRUE16-NEXT: s_not_b32 s13, s12
; GFX1364-TRUE16-NEXT: s_lshl_b32 s14, s2, s11
; GFX1364-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -13539,11 +13677,12 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1364-FAKE16-NEXT: s_clause 0x1
; GFX1364-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1364-FAKE16-NEXT: s_load_b32 s10, s[4:5], 0x34 nv
-; GFX1364-FAKE16-NEXT: s_mov_b64 s[6:7], exec
+; GFX1364-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1364-FAKE16-NEXT: s_mov_b64 s[4:5], exec
; GFX1364-FAKE16-NEXT: s_mov_b64 s[8:9], exec
-; GFX1364-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1364-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX1364-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1364-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1364-FAKE16-NEXT: ; implicit-def: $vgpr0
; GFX1364-FAKE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1364-FAKE16-NEXT: s_cbranch_execz .LBB16_4
@@ -13552,11 +13691,11 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1364-FAKE16-NEXT: s_and_b64 s[4:5], s[2:3], -4
; GFX1364-FAKE16-NEXT: s_and_b32 s2, s2, 3
; GFX1364-FAKE16-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1364-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1364-FAKE16-NEXT: s_sext_i32_i16 s7, s10
; GFX1364-FAKE16-NEXT: s_lshl_b32 s11, s2, 3
-; GFX1364-FAKE16-NEXT: s_mul_i32 s2, s10, s6
+; GFX1364-FAKE16-NEXT: s_mul_i32 s7, s7, s6
; GFX1364-FAKE16-NEXT: s_lshl_b32 s12, 0xffff, s11
-; GFX1364-FAKE16-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX1364-FAKE16-NEXT: s_and_b32 s2, 0xffff, s7
; GFX1364-FAKE16-NEXT: s_not_b32 s13, s12
; GFX1364-FAKE16-NEXT: s_lshl_b32 s14, s2, s11
; GFX1364-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -13600,12 +13739,12 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1332-TRUE16-NEXT: s_clause 0x1
; GFX1332-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1332-TRUE16-NEXT: s_load_b32 s8, s[4:5], 0x34 nv
-; GFX1332-TRUE16-NEXT: s_mov_b32 s6, exec_lo
+; GFX1332-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1332-TRUE16-NEXT: s_mov_b32 s4, exec_lo
; GFX1332-TRUE16-NEXT: s_mov_b32 s10, 0
-; GFX1332-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1332-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s4
; GFX1332-TRUE16-NEXT: s_mov_b32 s9, exec_lo
; GFX1332-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1332-TRUE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1332-TRUE16-NEXT: s_cbranch_execz .LBB16_4
; GFX1332-TRUE16-NEXT: ; %bb.1:
@@ -13613,11 +13752,11 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1332-TRUE16-NEXT: s_and_b64 s[4:5], s[2:3], -4
; GFX1332-TRUE16-NEXT: s_and_b32 s2, s2, 3
; GFX1332-TRUE16-NEXT: s_load_b32 s7, s[4:5], 0x0
-; GFX1332-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1332-TRUE16-NEXT: s_sext_i32_i16 s11, s8
; GFX1332-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1332-TRUE16-NEXT: s_mul_i32 s6, s8, s6
+; GFX1332-TRUE16-NEXT: s_mul_i32 s6, s11, s6
; GFX1332-TRUE16-NEXT: s_lshl_b32 s3, 0xffff, s2
-; GFX1332-TRUE16-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1332-TRUE16-NEXT: s_and_b32 s6, 0xffff, s6
; GFX1332-TRUE16-NEXT: s_not_b32 s11, s3
; GFX1332-TRUE16-NEXT: s_lshl_b32 s12, s6, s2
; GFX1332-TRUE16-NEXT: s_mov_b32 s6, -1
@@ -13659,12 +13798,12 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1332-FAKE16-NEXT: s_clause 0x1
; GFX1332-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1332-FAKE16-NEXT: s_load_b32 s8, s[4:5], 0x34 nv
-; GFX1332-FAKE16-NEXT: s_mov_b32 s6, exec_lo
+; GFX1332-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1332-FAKE16-NEXT: s_mov_b32 s4, exec_lo
; GFX1332-FAKE16-NEXT: s_mov_b32 s10, 0
-; GFX1332-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1332-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s4
; GFX1332-FAKE16-NEXT: s_mov_b32 s9, exec_lo
; GFX1332-FAKE16-NEXT: ; implicit-def: $vgpr0
-; GFX1332-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1332-FAKE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1332-FAKE16-NEXT: s_cbranch_execz .LBB16_4
; GFX1332-FAKE16-NEXT: ; %bb.1:
@@ -13672,11 +13811,11 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1332-FAKE16-NEXT: s_and_b64 s[4:5], s[2:3], -4
; GFX1332-FAKE16-NEXT: s_and_b32 s2, s2, 3
; GFX1332-FAKE16-NEXT: s_load_b32 s7, s[4:5], 0x0
-; GFX1332-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1332-FAKE16-NEXT: s_sext_i32_i16 s11, s8
; GFX1332-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1332-FAKE16-NEXT: s_mul_i32 s6, s8, s6
+; GFX1332-FAKE16-NEXT: s_mul_i32 s6, s11, s6
; GFX1332-FAKE16-NEXT: s_lshl_b32 s3, 0xffff, s2
-; GFX1332-FAKE16-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1332-FAKE16-NEXT: s_and_b32 s6, 0xffff, s6
; GFX1332-FAKE16-NEXT: s_not_b32 s11, s3
; GFX1332-FAKE16-NEXT: s_lshl_b32 s12, s6, s2
; GFX1332-FAKE16-NEXT: s_mov_b32 s6, -1
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
index b813e5b450e0a..567f3760e70b8 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
@@ -26,15 +26,15 @@ declare i32 @llvm.amdgcn.workitem.id.x()
define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out) {
; GFX7LESS-LABEL: add_i32_constant:
; GFX7LESS: ; %bb.0: ; %entry
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX7LESS-NEXT: ; implicit-def: $vgpr1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB0_2
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX7LESS-NEXT: s_mul_i32 s2, s2, 5
; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0
; GFX7LESS-NEXT: v_mov_b32_e32 v2, s2
@@ -54,15 +54,15 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out) {
;
; GFX8-LABEL: add_i32_constant:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB0_2
; GFX8-NEXT: ; %bb.1:
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX8-NEXT: s_mul_i32 s2, s2, 5
; GFX8-NEXT: v_mov_b32_e32 v1, 0
; GFX8-NEXT: v_mov_b32_e32 v2, s2
@@ -82,15 +82,15 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out) {
;
; GFX9-LABEL: add_i32_constant:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB0_2
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_mul_i32 s2, s2, 5
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: v_mov_b32_e32 v2, s2
@@ -109,17 +109,17 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out) {
;
; GFX1064-LABEL: add_i32_constant:
; GFX1064: ; %bb.0: ; %entry
-; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: ; implicit-def: $vgpr1
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB0_2
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-NEXT: s_mul_i32 s2, s2, 5
+; GFX1064-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-NEXT: v_mov_b32_e32 v2, s2
; GFX1064-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
@@ -138,16 +138,16 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out) {
;
; GFX1032-LABEL: add_i32_constant:
; GFX1032: ; %bb.0: ; %entry
-; GFX1032-NEXT: s_mov_b32 s1, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, exec_lo
; GFX1032-NEXT: ; implicit-def: $vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s1, s0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB0_2
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-NEXT: s_mul_i32 s1, s1, 5
+; GFX1032-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-NEXT: v_mov_b32_e32 v2, s1
; GFX1032-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
@@ -166,19 +166,19 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out) {
;
; GFX1164-LABEL: add_i32_constant:
; GFX1164: ; %bb.0: ; %entry
-; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX1164-NEXT: ; implicit-def: $vgpr1
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-NEXT: s_cbranch_execz .LBB0_2
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0
; GFX1164-NEXT: s_mul_i32 s2, s2, 5
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT: v_mov_b32_e32 v1, 0
; GFX1164-NEXT: v_mov_b32_e32 v2, s2
; GFX1164-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
@@ -196,17 +196,17 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out) {
;
; GFX1132-LABEL: add_i32_constant:
; GFX1132: ; %bb.0: ; %entry
-; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX1132-NEXT: ; implicit-def: $vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX1132-NEXT: s_mov_b32 s0, exec_lo
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB0_2
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1132-NEXT: s_mul_i32 s1, s1, 5
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s1
; GFX1132-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
@@ -224,19 +224,19 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out) {
;
; GFX1364-LABEL: add_i32_constant:
; GFX1364: ; %bb.0: ; %entry
-; GFX1364-NEXT: s_mov_b64 s[2:3], exec
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_mov_b64 s[0:1], exec
-; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX1364-NEXT: ; implicit-def: $vgpr1
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX1364-NEXT: s_mov_b64 s[0:1], exec
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1364-NEXT: s_cbranch_execz .LBB0_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1364-NEXT: v_mov_b32_e32 v1, 0
; GFX1364-NEXT: s_mul_i32 s2, s2, 5
-; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1364-NEXT: v_mov_b32_e32 v1, 0
; GFX1364-NEXT: v_mov_b32_e32 v2, s2
; GFX1364-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX1364-NEXT: s_wait_dscnt 0x0
@@ -254,17 +254,17 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out) {
;
; GFX1332-LABEL: add_i32_constant:
; GFX1332: ; %bb.0: ; %entry
-; GFX1332-NEXT: s_mov_b32 s1, exec_lo
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1332-NEXT: s_mov_b32 s0, exec_lo
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX1332-NEXT: ; implicit-def: $vgpr1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX1332-NEXT: s_mov_b32 s0, exec_lo
; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1332-NEXT: s_cbranch_execz .LBB0_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1332-NEXT: s_mul_i32 s1, s1, 5
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1332-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s1
; GFX1332-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX1332-NEXT: s_wait_dscnt 0x0
@@ -288,20 +288,20 @@ entry:
define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, i32 %additive) {
; GFX7LESS-LABEL: add_i32_uniform:
; GFX7LESS: ; %bb.0: ; %entry
-; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-NEXT: s_load_dword s2, s[4:5], 0xb
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX7LESS-NEXT: ; implicit-def: $vgpr1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB1_2
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_mul_i32 s2, s6, s2
+; GFX7LESS-NEXT: s_mul_i32 s3, s2, s3
; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, s2
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s3
; GFX7LESS-NEXT: s_mov_b32 m0, -1
; GFX7LESS-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
@@ -309,7 +309,7 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, i32 %additive)
; GFX7LESS-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX7LESS-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v1
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
@@ -319,20 +319,20 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, i32 %additive)
;
; GFX8-LABEL: add_i32_uniform:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB1_2
; GFX8-NEXT: ; %bb.1:
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mul_i32 s2, s6, s2
+; GFX8-NEXT: s_mul_i32 s3, s2, s3
; GFX8-NEXT: v_mov_b32_e32 v1, 0
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: v_mov_b32_e32 v2, s3
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
@@ -340,7 +340,7 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, i32 %additive)
; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX8-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX8-NEXT: v_readfirstlane_b32 s4, v1
; GFX8-NEXT: s_mov_b32 s3, 0xf000
; GFX8-NEXT: s_mov_b32 s2, -1
@@ -350,27 +350,27 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, i32 %additive)
;
; GFX9-LABEL: add_i32_uniform:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB1_2
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_i32 s2, s6, s2
+; GFX9-NEXT: s_mul_i32 s3, s2, s3
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_mov_b32_e32 v2, s2
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
; GFX9-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: .LBB1_2:
; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX9-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX9-NEXT: v_readfirstlane_b32 s4, v1
; GFX9-NEXT: s_mov_b32 s3, 0xf000
; GFX9-NEXT: s_mov_b32 s2, -1
@@ -380,20 +380,20 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, i32 %additive)
;
; GFX1064-LABEL: add_i32_uniform:
; GFX1064: ; %bb.0: ; %entry
-; GFX1064-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: ; implicit-def: $vgpr1
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB1_2
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_mul_i32 s2, s6, s2
-; GFX1064-NEXT: v_mov_b32_e32 v2, s2
+; GFX1064-NEXT: s_mul_i32 s3, s2, s3
+; GFX1064-NEXT: v_mov_b32_e32 v1, 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, s3
; GFX1064-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: buffer_gl0_inv
@@ -401,9 +401,10 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, i32 %additive)
; GFX1064-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX1064-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: v_readfirstlane_b32 s2, v1
+; GFX1064-NEXT: s_mov_b32 null, 0
+; GFX1064-NEXT: v_readfirstlane_b32 s4, v1
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s6, v0, s[2:3]
+; GFX1064-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s2, v0, s[4:5]
; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
; GFX1064-NEXT: s_mov_b32 s2, -1
; GFX1064-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -412,17 +413,17 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, i32 %additive)
; GFX1032-LABEL: add_i32_uniform:
; GFX1032: ; %bb.0: ; %entry
; GFX1032-NEXT: s_load_dword s0, s[4:5], 0x2c
-; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s1, exec_lo
; GFX1032-NEXT: ; implicit-def: $vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s2, s1
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB1_2
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s2, s2
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_mul_i32 s2, s0, s2
+; GFX1032-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-NEXT: v_mov_b32_e32 v2, s2
; GFX1032-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
@@ -441,33 +442,32 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, i32 %additive)
;
; GFX1164-LABEL: add_i32_uniform:
; GFX1164: ; %bb.0: ; %entry
-; GFX1164-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX1164-NEXT: ; implicit-def: $vgpr1
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-NEXT: s_cbranch_execz .LBB1_2
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_mul_i32 s2, s6, s2
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: v_mov_b32_e32 v2, s2
+; GFX1164-NEXT: s_mul_i32 s3, s2, s3
+; GFX1164-NEXT: v_mov_b32_e32 v1, 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, s3
; GFX1164-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: buffer_gl0_inv
; GFX1164-NEXT: .LBB1_2:
; GFX1164-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_readfirstlane_b32 s2, v1
-; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_mad_u64_u32 v[1:2], null, s6, v0, s[2:3]
+; GFX1164-NEXT: v_readfirstlane_b32 s4, v1
; GFX1164-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: v_mad_u64_u32 v[1:2], null, s2, v0, s[4:5]
; GFX1164-NEXT: s_mov_b32 s2, -1
; GFX1164-NEXT: buffer_store_b32 v1, off, s[0:3], 0
; GFX1164-NEXT: s_endpgm
@@ -475,15 +475,15 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, i32 %additive)
; GFX1132-LABEL: add_i32_uniform:
; GFX1132: ; %bb.0: ; %entry
; GFX1132-NEXT: s_load_b32 s0, s[4:5], 0x2c
-; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX1132-NEXT: ; implicit-def: $vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB1_2
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_mul_i32 s2, s0, s2
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -504,33 +504,32 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, i32 %additive)
;
; GFX1364-LABEL: add_i32_uniform:
; GFX1364: ; %bb.0: ; %entry
-; GFX1364-NEXT: s_load_b32 s6, s[4:5], 0x2c nv
-; GFX1364-NEXT: s_mov_b64 s[2:3], exec
+; GFX1364-NEXT: s_load_b32 s2, s[4:5], 0x2c nv
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_mov_b64 s[0:1], exec
-; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX1364-NEXT: ; implicit-def: $vgpr1
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX1364-NEXT: s_mov_b64 s[0:1], exec
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1364-NEXT: s_cbranch_execz .LBB1_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1364-NEXT: v_mov_b32_e32 v1, 0
; GFX1364-NEXT: s_wait_kmcnt 0x0
-; GFX1364-NEXT: s_mul_i32 s2, s6, s2
-; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1364-NEXT: v_mov_b32_e32 v2, s2
+; GFX1364-NEXT: s_mul_i32 s3, s2, s3
+; GFX1364-NEXT: v_mov_b32_e32 v1, 0
+; GFX1364-NEXT: v_mov_b32_e32 v2, s3
; GFX1364-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX1364-NEXT: s_wait_dscnt 0x0
; GFX1364-NEXT: global_inv scope:SCOPE_SE
; GFX1364-NEXT: .LBB1_2:
; GFX1364-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX1364-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX1364-NEXT: v_readfirstlane_b32 s2, v1
-; GFX1364-NEXT: s_wait_kmcnt 0x0
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1364-NEXT: v_mad_co_u64_u32 v[0:1], null, s6, v0, s[2:3]
+; GFX1364-NEXT: v_readfirstlane_b32 s4, v1
; GFX1364-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1364-NEXT: s_wait_kmcnt 0x0
+; GFX1364-NEXT: v_mad_co_u64_u32 v[0:1], null, s2, v0, s[4:5]
; GFX1364-NEXT: s_mov_b32 s2, -1
; GFX1364-NEXT: buffer_store_b32 v0, off, s[0:3], null
; GFX1364-NEXT: s_endpgm
@@ -538,15 +537,15 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, i32 %additive)
; GFX1332-LABEL: add_i32_uniform:
; GFX1332: ; %bb.0: ; %entry
; GFX1332-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
-; GFX1332-NEXT: s_mov_b32 s2, exec_lo
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1332-NEXT: s_mov_b32 s1, exec_lo
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX1332-NEXT: ; implicit-def: $vgpr1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX1332-NEXT: s_mov_b32 s1, exec_lo
; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1332-NEXT: s_cbranch_execz .LBB1_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX1332-NEXT: s_wait_kmcnt 0x0
; GFX1332-NEXT: s_mul_i32 s2, s0, s2
; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -1280,112 +1279,102 @@ entry:
define amdgpu_kernel void @add_i32_varying_nouse() {
; GFX7LESS_ITERATIVE-LABEL: add_i32_varying_nouse:
; GFX7LESS_ITERATIVE: ; %bb.0: ; %entry
-; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s2, 0
-; GFX7LESS_ITERATIVE-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX7LESS_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS_ITERATIVE-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX7LESS_ITERATIVE-NEXT: s_ff1_i32_b64 s3, s[0:1]
-; GFX7LESS_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s3
-; GFX7LESS_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS_ITERATIVE-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
-; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s3
-; GFX7LESS_ITERATIVE-NEXT: s_add_i32 s2, s2, s6
-; GFX7LESS_ITERATIVE-NEXT: s_and_b64 vcc, exec, s[4:5]
-; GFX7LESS_ITERATIVE-NEXT: s_cbranch_vccnz .LBB3_1
-; GFX7LESS_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s4, v0, s3
+; GFX7LESS_ITERATIVE-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX7LESS_ITERATIVE-NEXT: s_add_i32 s2, s2, s4
+; GFX7LESS_ITERATIVE-NEXT: s_or_b64 s[4:5], s[0:1], s[0:1]
+; GFX7LESS_ITERATIVE-NEXT: s_cbranch_scc1 .LBB3_1
+; GFX7LESS_ITERATIVE-NEXT: ; %bb.2:
+; GFX7LESS_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS_ITERATIVE-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS_ITERATIVE-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS_ITERATIVE-NEXT: s_cbranch_execz .LBB3_4
; GFX7LESS_ITERATIVE-NEXT: ; %bb.3:
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v1, s2
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v0, s2
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 m0, -1
-; GFX7LESS_ITERATIVE-NEXT: ds_add_u32 v0, v1
+; GFX7LESS_ITERATIVE-NEXT: ds_add_u32 v1, v0
; GFX7LESS_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS_ITERATIVE-NEXT: .LBB3_4:
; GFX7LESS_ITERATIVE-NEXT: s_endpgm
;
; GFX8_ITERATIVE-LABEL: add_i32_varying_nouse:
; GFX8_ITERATIVE: ; %bb.0: ; %entry
-; GFX8_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX8_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX8_ITERATIVE-NEXT: s_mov_b32 s2, 0
-; GFX8_ITERATIVE-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX8_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX8_ITERATIVE-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX8_ITERATIVE-NEXT: s_ff1_i32_b64 s3, s[0:1]
; GFX8_ITERATIVE-NEXT: v_readlane_b32 s4, v0, s3
+; GFX8_ITERATIVE-NEXT: s_bitset0_b64 s[0:1], s3
; GFX8_ITERATIVE-NEXT: s_add_i32 s2, s2, s4
-; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s3
-; GFX8_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
+; GFX8_ITERATIVE-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX8_ITERATIVE-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX8_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX8_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX8_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX8_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8_ITERATIVE-NEXT: ; %bb.2:
+; GFX8_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX8_ITERATIVE-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX8_ITERATIVE-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX8_ITERATIVE-NEXT: s_cbranch_execz .LBB3_4
; GFX8_ITERATIVE-NEXT: ; %bb.3:
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v0, 0
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v1, s2
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v0, s2
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
; GFX8_ITERATIVE-NEXT: s_mov_b32 m0, -1
-; GFX8_ITERATIVE-NEXT: ds_add_u32 v0, v1
+; GFX8_ITERATIVE-NEXT: ds_add_u32 v1, v0
; GFX8_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX8_ITERATIVE-NEXT: .LBB3_4:
; GFX8_ITERATIVE-NEXT: s_endpgm
;
; GFX9_ITERATIVE-LABEL: add_i32_varying_nouse:
; GFX9_ITERATIVE: ; %bb.0: ; %entry
-; GFX9_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX9_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX9_ITERATIVE-NEXT: s_mov_b32 s2, 0
-; GFX9_ITERATIVE-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX9_ITERATIVE-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX9_ITERATIVE-NEXT: s_ff1_i32_b64 s3, s[0:1]
; GFX9_ITERATIVE-NEXT: v_readlane_b32 s4, v0, s3
+; GFX9_ITERATIVE-NEXT: s_bitset0_b64 s[0:1], s3
; GFX9_ITERATIVE-NEXT: s_add_i32 s2, s2, s4
-; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s3
-; GFX9_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
+; GFX9_ITERATIVE-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX9_ITERATIVE-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX9_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9_ITERATIVE-NEXT: ; %bb.2:
+; GFX9_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9_ITERATIVE-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9_ITERATIVE-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9_ITERATIVE-NEXT: s_cbranch_execz .LBB3_4
; GFX9_ITERATIVE-NEXT: ; %bb.3:
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v0, 0
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v1, s2
-; GFX9_ITERATIVE-NEXT: ds_add_u32 v0, v1
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v0, s2
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
+; GFX9_ITERATIVE-NEXT: ds_add_u32 v1, v0
; GFX9_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX9_ITERATIVE-NEXT: .LBB3_4:
; GFX9_ITERATIVE-NEXT: s_endpgm
;
; GFX1064_ITERATIVE-LABEL: add_i32_varying_nouse:
; GFX1064_ITERATIVE: ; %bb.0: ; %entry
-; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064_ITERATIVE-NEXT: s_mov_b32 s2, 0
-; GFX1064_ITERATIVE-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1064_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064_ITERATIVE-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX1064_ITERATIVE-NEXT: s_ff1_i32_b64 s3, s[0:1]
-; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1064_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s3
-; GFX1064_ITERATIVE-NEXT: s_add_i32 s2, s2, s6
-; GFX1064_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
+; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1064_ITERATIVE-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1064_ITERATIVE-NEXT: s_add_i32 s2, s2, s4
+; GFX1064_ITERATIVE-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX1064_ITERATIVE-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1064_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064_ITERATIVE-NEXT: ; %bb.2:
+; GFX1064_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064_ITERATIVE-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064_ITERATIVE-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064_ITERATIVE-NEXT: s_cbranch_execz .LBB3_4
; GFX1064_ITERATIVE-NEXT: ; %bb.3:
-; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v0, 0
-; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064_ITERATIVE-NEXT: ds_add_u32 v0, v1
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v0, s2
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
+; GFX1064_ITERATIVE-NEXT: ds_add_u32 v1, v0
; GFX1064_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064_ITERATIVE-NEXT: buffer_gl0_inv
; GFX1064_ITERATIVE-NEXT: .LBB3_4:
@@ -1393,26 +1382,24 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
;
; GFX1032_ITERATIVE-LABEL: add_i32_varying_nouse:
; GFX1032_ITERATIVE: ; %bb.0: ; %entry
-; GFX1032_ITERATIVE-NEXT: s_mov_b32 s1, exec_lo
+; GFX1032_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1032_ITERATIVE-NEXT: s_mov_b32 s0, 0
-; GFX1032_ITERATIVE-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1032_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032_ITERATIVE-NEXT: s_mov_b32 s1, exec_lo
+; GFX1032_ITERATIVE-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX1032_ITERATIVE-NEXT: s_ff1_i32_b32 s2, s1
; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s2, 1, s2
+; GFX1032_ITERATIVE-NEXT: s_bitset0_b32 s1, s2
; GFX1032_ITERATIVE-NEXT: s_add_i32 s0, s0, s3
-; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s1, s1, s2
+; GFX1032_ITERATIVE-NEXT: s_cmp_lg_u32 s1, 0
; GFX1032_ITERATIVE-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1032_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032_ITERATIVE-NEXT: ; %bb.2:
+; GFX1032_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032_ITERATIVE-NEXT: s_and_saveexec_b32 s1, vcc_lo
-; GFX1032_ITERATIVE-NEXT: s_xor_b32 s1, exec_lo, s1
; GFX1032_ITERATIVE-NEXT: s_cbranch_execz .LBB3_4
; GFX1032_ITERATIVE-NEXT: ; %bb.3:
-; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v0, 0
-; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v1, s0
-; GFX1032_ITERATIVE-NEXT: ds_add_u32 v0, v1
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
+; GFX1032_ITERATIVE-NEXT: ds_add_u32 v1, v0
; GFX1032_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032_ITERATIVE-NEXT: buffer_gl0_inv
; GFX1032_ITERATIVE-NEXT: .LBB3_4:
@@ -1420,30 +1407,28 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
;
; GFX1164_ITERATIVE-LABEL: add_i32_varying_nouse:
; GFX1164_ITERATIVE: ; %bb.0: ; %entry
+; GFX1164_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
; GFX1164_ITERATIVE-NEXT: s_mov_b32 s2, 0
-; GFX1164_ITERATIVE-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1164_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1164_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164_ITERATIVE-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX1164_ITERATIVE-NEXT: s_ctz_i32_b64 s3, s[0:1]
-; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s3
-; GFX1164_ITERATIVE-NEXT: s_add_i32 s2, s2, s6
-; GFX1164_ITERATIVE-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[4:5]
+; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1164_ITERATIVE-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164_ITERATIVE-NEXT: s_add_i32 s2, s2, s4
+; GFX1164_ITERATIVE-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX1164_ITERATIVE-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1164_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164_ITERATIVE-NEXT: ; %bb.2:
; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164_ITERATIVE-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164_ITERATIVE-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164_ITERATIVE-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164_ITERATIVE-NEXT: s_cbranch_execz .LBB3_4
; GFX1164_ITERATIVE-NEXT: ; %bb.3:
-; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v0, 0
-; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v1, s2
-; GFX1164_ITERATIVE-NEXT: ds_add_u32 v0, v1
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v0, s2
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
+; GFX1164_ITERATIVE-NEXT: ds_add_u32 v1, v0
; GFX1164_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164_ITERATIVE-NEXT: buffer_gl0_inv
; GFX1164_ITERATIVE-NEXT: .LBB3_4:
@@ -1451,28 +1436,26 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
;
; GFX1132_ITERATIVE-LABEL: add_i32_varying_nouse:
; GFX1132_ITERATIVE: ; %bb.0: ; %entry
+; GFX1132_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1132_ITERATIVE-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1132_ITERATIVE-NEXT: s_mov_b32 s1, exec_lo
; GFX1132_ITERATIVE-NEXT: s_mov_b32 s0, 0
-; GFX1132_ITERATIVE-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1132_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132_ITERATIVE-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132_ITERATIVE-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1132_ITERATIVE-NEXT: s_ctz_i32_b32 s2, s1
; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s2, 1, s2
+; GFX1132_ITERATIVE-NEXT: s_bitset0_b32 s1, s2
; GFX1132_ITERATIVE-NEXT: s_add_i32 s0, s0, s3
-; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s1, s1, s2
+; GFX1132_ITERATIVE-NEXT: s_cmp_lg_u32 s1, 0
; GFX1132_ITERATIVE-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1132_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132_ITERATIVE-NEXT: ; %bb.2:
; GFX1132_ITERATIVE-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132_ITERATIVE-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132_ITERATIVE-NEXT: s_xor_b32 s1, exec_lo, s1
+; GFX1132_ITERATIVE-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132_ITERATIVE-NEXT: s_cbranch_execz .LBB3_4
; GFX1132_ITERATIVE-NEXT: ; %bb.3:
-; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
-; GFX1132_ITERATIVE-NEXT: ds_add_u32 v0, v1
+; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, 0
+; GFX1132_ITERATIVE-NEXT: ds_add_u32 v1, v0
; GFX1132_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132_ITERATIVE-NEXT: buffer_gl0_inv
; GFX1132_ITERATIVE-NEXT: .LBB3_4:
@@ -1488,31 +1471,31 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
;
; GFX8_DPP-LABEL: add_i32_varying_nouse:
; GFX8_DPP: ; %bb.0: ; %entry
-; GFX8_DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX8_DPP-NEXT: v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX8_DPP-NEXT: v_mbcnt_hi_u32_b32 v3, exec_hi, v3
+; GFX8_DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX8_DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
; GFX8_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX8_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]
; GFX8_DPP-NEXT: s_nop 1
-; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX8_DPP-NEXT: s_nop 1
-; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX8_DPP-NEXT: s_nop 1
-; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX8_DPP-NEXT: s_nop 1
-; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX8_DPP-NEXT: s_nop 1
-; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX8_DPP-NEXT: s_nop 1
-; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX8_DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX8_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX8_DPP-NEXT: s_mov_b32 s0, s2
-; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX8_DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX8_DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX8_DPP-NEXT: ; %bb.1:
; GFX8_DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX8_DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX8_DPP-NEXT: s_mov_b32 m0, -1
; GFX8_DPP-NEXT: ds_add_u32 v2, v0
; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -1521,31 +1504,31 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
;
; GFX9_DPP-LABEL: add_i32_varying_nouse:
; GFX9_DPP: ; %bb.0: ; %entry
-; GFX9_DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX9_DPP-NEXT: v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9_DPP-NEXT: v_mbcnt_hi_u32_b32 v3, exec_hi, v3
+; GFX9_DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9_DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
; GFX9_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]
; GFX9_DPP-NEXT: s_nop 1
-; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9_DPP-NEXT: s_nop 1
-; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9_DPP-NEXT: s_nop 1
-; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9_DPP-NEXT: s_nop 1
-; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9_DPP-NEXT: s_nop 1
-; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9_DPP-NEXT: s_nop 1
-; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9_DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX9_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9_DPP-NEXT: s_mov_b32 s0, s2
-; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX9_DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9_DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX9_DPP-NEXT: ; %bb.1:
; GFX9_DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX9_DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9_DPP-NEXT: ds_add_u32 v2, v0
; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9_DPP-NEXT: .LBB3_2:
@@ -1555,27 +1538,34 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
; GFX1064_DPP: ; %bb.0: ; %entry
; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]
-; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1064_DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1064_DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064_DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064_DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064_DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1064_DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
; GFX1064_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064_DPP-NEXT: v_readlane_b32 s2, v1, 0
-; GFX1064_DPP-NEXT: v_readlane_b32 s3, v1, 32
+; GFX1064_DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX1064_DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v3, exec_hi, v0
-; GFX1064_DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX1064_DPP-NEXT: s_add_i32 s0, s2, s3
-; GFX1064_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064_DPP-NEXT: s_mov_b32 s0, s2
+; GFX1064_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064_DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064_DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1064_DPP-NEXT: ; %bb.1:
-; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, s0
-; GFX1064_DPP-NEXT: ds_add_u32 v0, v3
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1064_DPP-NEXT: ds_add_u32 v4, v0
; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064_DPP-NEXT: buffer_gl0_inv
; GFX1064_DPP-NEXT: .LBB3_2:
@@ -1585,21 +1575,24 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
; GFX1032_DPP: ; %bb.0: ; %entry
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
-; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1032_DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032_DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1032_DPP-NEXT: v_readlane_b32 s1, v1, 31
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, v1
-; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v4
-; GFX1032_DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032_DPP-NEXT: s_mov_b32 s0, s1
+; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032_DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032_DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1032_DPP-NEXT: ; %bb.1:
-; GFX1032_DPP-NEXT: ds_add_u32 v0, v3
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032_DPP-NEXT: ds_add_u32 v3, v0
; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032_DPP-NEXT: buffer_gl0_inv
; GFX1032_DPP-NEXT: .LBB3_2:
@@ -1611,33 +1604,41 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164_DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1164_DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164_DPP-NEXT: v_permlane64_b32 v2, v1
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164_DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, v1
-; GFX1164_DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1164_DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164_DPP-NEXT: s_mov_b32 s0, s2
+; GFX1164_DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164_DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164_DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1164_DPP-NEXT: ; %bb.1:
-; GFX1164_DPP-NEXT: ds_add_u32 v0, v3
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164_DPP-NEXT: ds_add_u32 v4, v0
; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164_DPP-NEXT: buffer_gl0_inv
; GFX1164_DPP-NEXT: .LBB3_2:
@@ -1650,24 +1651,26 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132_DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132_DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: v_readlane_b32 s1, v1, 31
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132_DPP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, v1
-; GFX1132_DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132_DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132_DPP-NEXT: s_mov_b32 s0, s1
+; GFX1132_DPP-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132_DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132_DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1132_DPP-NEXT: ; %bb.1:
-; GFX1132_DPP-NEXT: ds_add_u32 v0, v3
+; GFX1132_DPP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, 0
+; GFX1132_DPP-NEXT: ds_add_u32 v3, v0
; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132_DPP-NEXT: buffer_gl0_inv
; GFX1132_DPP-NEXT: .LBB3_2:
@@ -1679,33 +1682,40 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
; GFX1364-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1364-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1364-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1364-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1364-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1364-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1364-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1364-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1364-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1364-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1364-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1364-NEXT: s_wait_dscnt 0x0
+; GFX1364-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1364-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1364-NEXT: s_wait_dscnt 0x0
; GFX1364-NEXT: v_add_nc_u32_e32 v1, v1, v2
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1364-NEXT: v_permlane64_b32 v2, v1
; GFX1364-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1364-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1364-NEXT: v_readlane_b32 s2, v1, 63
; GFX1364-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1364-NEXT: v_mov_b32_e32 v0, 0
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1364-NEXT: v_mov_b32_e32 v3, v1
-; GFX1364-NEXT: s_mov_b64 s[0:1], exec
-; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1364-NEXT: s_mov_b32 s0, s2
+; GFX1364-NEXT: s_mov_b64 s[2:3], exec
+; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1364-NEXT: s_cbranch_execz .LBB3_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: ds_add_u32 v0, v3
+; GFX1364-NEXT: v_mov_b32_e32 v0, s0
+; GFX1364-NEXT: v_mov_b32_e32 v4, 0
+; GFX1364-NEXT: ds_add_u32 v4, v0
; GFX1364-NEXT: s_wait_dscnt 0x0
; GFX1364-NEXT: global_inv scope:SCOPE_SE
; GFX1364-NEXT: .LBB3_2:
@@ -1718,24 +1728,26 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1332-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1332-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1332-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1332-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1332-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1332-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1332-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1332-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1332-NEXT: s_wait_dscnt 0x0
; GFX1332-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1332-NEXT: v_readlane_b32 s1, v1, 31
; GFX1332-NEXT: s_mov_b32 exec_lo, s0
-; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1332-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, v1
-; GFX1332-NEXT: s_mov_b32 s0, exec_lo
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1332-NEXT: s_mov_b32 s0, s1
+; GFX1332-NEXT: s_mov_b32 s1, exec_lo
+; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1332-NEXT: s_cbranch_execz .LBB3_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: ds_add_u32 v0, v3
+; GFX1332-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, 0
+; GFX1332-NEXT: ds_add_u32 v3, v0
; GFX1332-NEXT: s_wait_dscnt 0x0
; GFX1332-NEXT: global_inv scope:SCOPE_SE
; GFX1332-NEXT: .LBB3_2:
@@ -1749,20 +1761,25 @@ entry:
define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out) {
; GFX7LESS-LABEL: add_i64_constant:
; GFX7LESS: ; %bb.0: ; %entry
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, s3, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX7LESS-NEXT: v_mul_hi_u32 v0, 5, s2
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v1
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: v_readfirstlane_b32 s3, v0
; GFX7LESS-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB4_2
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-NEXT: s_mul_i32 s2, s2, 5
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0
+; GFX7LESS-NEXT: s_mul_i32 s6, s2, 5
+; GFX7LESS-NEXT: s_mul_i32 s2, s2, 0
+; GFX7LESS-NEXT: s_add_u32 s7, s3, s2
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, 0
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s7
; GFX7LESS-NEXT: s_mov_b32 m0, -1
-; GFX7LESS-NEXT: ds_add_rtn_u64 v[0:1], v1, v[0:1]
+; GFX7LESS-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: .LBB4_2:
; GFX7LESS-NEXT: s_or_b64 exec, exec, s[0:1]
@@ -1782,20 +1799,25 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out) {
;
; GFX8-LABEL: add_i64_constant:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, s3, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX8-NEXT: v_mul_hi_u32 v0, 5, s2
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX8-NEXT: v_readfirstlane_b32 s3, v0
; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB4_2
; GFX8-NEXT: ; %bb.1:
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX8-NEXT: s_mul_i32 s2, s2, 5
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_mov_b32_e32 v1, 0
+; GFX8-NEXT: s_mul_i32 s6, s2, 5
+; GFX8-NEXT: s_mul_i32 s2, s2, 0
+; GFX8-NEXT: s_add_u32 s7, s3, s2
+; GFX8-NEXT: v_mov_b32_e32 v3, 0
+; GFX8-NEXT: v_mov_b32_e32 v0, s6
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_mov_b32 m0, -1
-; GFX8-NEXT: ds_add_rtn_u64 v[0:1], v1, v[0:1]
+; GFX8-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: .LBB4_2:
; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
@@ -1815,19 +1837,23 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out) {
;
; GFX9-LABEL: add_i64_constant:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, s3, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB4_2
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX9-NEXT: s_mul_i32 s2, s2, 5
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: ds_add_rtn_u64 v[0:1], v1, v[0:1]
+; GFX9-NEXT: s_mul_i32 s6, s2, 5
+; GFX9-NEXT: s_mul_hi_u32 s3, 5, s2
+; GFX9-NEXT: s_mul_i32 s2, s2, 0
+; GFX9-NEXT: s_add_u32 s7, s3, s2
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: .LBB4_2:
; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
@@ -1846,19 +1872,23 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out) {
;
; GFX1064-LABEL: add_i64_constant:
; GFX1064: ; %bb.0: ; %entry
-; GFX1064-NEXT: s_mov_b64 s[2:3], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, s3, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1064-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB4_2
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-NEXT: v_mov_b32_e32 v1, 0
+; GFX1064-NEXT: s_mul_hi_u32 s3, 5, s2
+; GFX1064-NEXT: s_mul_i32 s6, s2, 0
; GFX1064-NEXT: s_mul_i32 s2, s2, 5
+; GFX1064-NEXT: s_add_u32 s3, s3, s6
+; GFX1064-NEXT: v_mov_b32_e32 v3, 0
; GFX1064-NEXT: v_mov_b32_e32 v0, s2
-; GFX1064-NEXT: ds_add_rtn_u64 v[0:1], v1, v[0:1]
+; GFX1064-NEXT: v_mov_b32_e32 v1, s3
+; GFX1064-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: buffer_gl0_inv
; GFX1064-NEXT: .LBB4_2:
@@ -1876,18 +1906,22 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out) {
;
; GFX1032-LABEL: add_i64_constant:
; GFX1032: ; %bb.0: ; %entry
-; GFX1032-NEXT: s_mov_b32 s1, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, exec_lo
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, s1, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s1, s0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB4_2
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0
-; GFX1032-NEXT: s_mul_i32 s1, s1, 5
-; GFX1032-NEXT: v_mov_b32_e32 v0, s1
-; GFX1032-NEXT: ds_add_rtn_u64 v[0:1], v1, v[0:1]
+; GFX1032-NEXT: s_mul_hi_u32 s3, 5, s1
+; GFX1032-NEXT: s_mul_i32 s6, s1, 0
+; GFX1032-NEXT: s_mul_i32 s2, s1, 5
+; GFX1032-NEXT: s_add_u32 s3, s3, s6
+; GFX1032-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-NEXT: v_mov_b32_e32 v0, s2
+; GFX1032-NEXT: v_mov_b32_e32 v1, s3
+; GFX1032-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: buffer_gl0_inv
; GFX1032-NEXT: .LBB4_2:
@@ -1905,21 +1939,25 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out) {
;
; GFX1164-LABEL: add_i64_constant:
; GFX1164: ; %bb.0: ; %entry
-; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, s3, v0
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1164-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB4_2
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0
+; GFX1164-NEXT: s_mul_hi_u32 s3, 5, s2
+; GFX1164-NEXT: s_mul_i32 s6, s2, 0
; GFX1164-NEXT: s_mul_i32 s2, s2, 5
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT: s_add_u32 s3, s3, s6
+; GFX1164-NEXT: v_mov_b32_e32 v3, 0
; GFX1164-NEXT: v_mov_b32_e32 v0, s2
-; GFX1164-NEXT: ds_add_rtn_u64 v[0:1], v1, v[0:1]
+; GFX1164-NEXT: v_mov_b32_e32 v1, s3
+; GFX1164-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: buffer_gl0_inv
; GFX1164-NEXT: .LBB4_2:
@@ -1937,20 +1975,22 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out) {
;
; GFX1132-LABEL: add_i64_constant:
; GFX1132: ; %bb.0: ; %entry
-; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, s1, 0
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX1132-NEXT: s_mov_b32 s0, exec_lo
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB4_2
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX1132-NEXT: v_mov_b32_e32 v1, 0
-; GFX1132-NEXT: s_mul_i32 s1, s1, 5
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_mov_b32_e32 v0, s1
-; GFX1132-NEXT: ds_add_rtn_u64 v[0:1], v1, v[0:1]
+; GFX1132-NEXT: s_mul_hi_u32 s3, 5, s1
+; GFX1132-NEXT: s_mul_i32 s6, s1, 0
+; GFX1132-NEXT: s_mul_i32 s2, s1, 5
+; GFX1132-NEXT: s_add_u32 s3, s3, s6
+; GFX1132-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s2
+; GFX1132-NEXT: v_mov_b32_e32 v1, s3
+; GFX1132-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: buffer_gl0_inv
; GFX1132-NEXT: .LBB4_2:
@@ -1968,21 +2008,25 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out) {
;
; GFX1364-LABEL: add_i64_constant:
; GFX1364: ; %bb.0: ; %entry
-; GFX1364-NEXT: s_mov_b64 s[2:3], exec
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_mov_b64 s[0:1], exec
-; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v2, s3, v0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX1364-NEXT: s_mov_b64 s[0:1], exec
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1364-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1364-NEXT: s_cbranch_execz .LBB4_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1364-NEXT: v_mov_b32_e32 v1, 0
+; GFX1364-NEXT: s_mul_hi_u32 s3, 5, s2
+; GFX1364-NEXT: s_mul_i32 s6, s2, 0
; GFX1364-NEXT: s_mul_i32 s2, s2, 5
-; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1364-NEXT: s_add_co_u32 s3, s3, s6
+; GFX1364-NEXT: v_mov_b32_e32 v3, 0
; GFX1364-NEXT: v_mov_b32_e32 v0, s2
-; GFX1364-NEXT: ds_add_rtn_u64 v[0:1], v1, v[0:1]
+; GFX1364-NEXT: v_mov_b32_e32 v1, s3
+; GFX1364-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
; GFX1364-NEXT: s_wait_dscnt 0x0
; GFX1364-NEXT: global_inv scope:SCOPE_SE
; GFX1364-NEXT: .LBB4_2:
@@ -2000,20 +2044,22 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out) {
;
; GFX1332-LABEL: add_i64_constant:
; GFX1332: ; %bb.0: ; %entry
-; GFX1332-NEXT: s_mov_b32 s1, exec_lo
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1332-NEXT: s_mov_b32 s0, exec_lo
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v2, s1, 0
; GFX1332-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX1332-NEXT: s_mov_b32 s0, exec_lo
; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1332-NEXT: s_cbranch_execz .LBB4_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX1332-NEXT: v_mov_b32_e32 v1, 0
-; GFX1332-NEXT: s_mul_i32 s1, s1, 5
-; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1332-NEXT: v_mov_b32_e32 v0, s1
-; GFX1332-NEXT: ds_add_rtn_u64 v[0:1], v1, v[0:1]
+; GFX1332-NEXT: s_mul_hi_u32 s3, 5, s1
+; GFX1332-NEXT: s_mul_i32 s6, s1, 0
+; GFX1332-NEXT: s_mul_i32 s2, s1, 5
+; GFX1332-NEXT: s_add_co_u32 s3, s3, s6
+; GFX1332-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s2
+; GFX1332-NEXT: v_mov_b32_e32 v1, s3
+; GFX1332-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
; GFX1332-NEXT: s_wait_dscnt 0x0
; GFX1332-NEXT: global_inv scope:SCOPE_SE
; GFX1332-NEXT: .LBB4_2:
@@ -2038,29 +2084,30 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, i64 %additive)
; GFX7LESS-LABEL: add_i64_uniform:
; GFX7LESS: ; %bb.0: ; %entry
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, s7, v0
+; GFX7LESS-NEXT: s_mov_b64 s[4:5], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mul_hi_u32 v0, s2, v0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v1
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: v_readfirstlane_b32 s7, v0
; GFX7LESS-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB5_2
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mul_hi_u32 v0, s2, v0
-; GFX7LESS-NEXT: s_mul_i32 s7, s3, s6
-; GFX7LESS-NEXT: s_mul_i32 s6, s2, s6
+; GFX7LESS-NEXT: s_mul_i32 s8, s2, s6
+; GFX7LESS-NEXT: s_mul_i32 s6, s3, s6
+; GFX7LESS-NEXT: s_add_u32 s9, s7, s6
; GFX7LESS-NEXT: v_mov_b32_e32 v3, 0
-; GFX7LESS-NEXT: v_add_i32_e32 v1, vcc, s7, v0
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s8
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s9
; GFX7LESS-NEXT: s_mov_b32 m0, -1
; GFX7LESS-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: .LBB5_2:
; GFX7LESS-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_mov_b32 s4, s0
; GFX7LESS-NEXT: v_readfirstlane_b32 s0, v1
; GFX7LESS-NEXT: v_mul_lo_u32 v1, s3, v2
@@ -2080,22 +2127,26 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, i64 %additive)
; GFX8-LABEL: add_i64_uniform:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b64 s[6:7], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX8-NEXT: s_mov_b64 s[4:5], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, s6
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mul_hi_u32 v0, s2, v0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX8-NEXT: v_readfirstlane_b32 s7, v0
; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX8-NEXT: s_cbranch_execz .LBB5_2
; GFX8-NEXT: ; %bb.1:
-; GFX8-NEXT: s_bcnt1_i32_b64 s8, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v0, s8
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[6:7], s2, v0, 0
-; GFX8-NEXT: s_mul_i32 s6, s3, s8
+; GFX8-NEXT: s_mul_i32 s8, s2, s6
+; GFX8-NEXT: s_mul_i32 s6, s3, s6
+; GFX8-NEXT: s_add_u32 s9, s7, s6
; GFX8-NEXT: v_mov_b32_e32 v3, 0
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
+; GFX8-NEXT: v_mov_b32_e32 v1, s9
; GFX8-NEXT: s_mov_b32 m0, -1
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, s6, v1
; GFX8-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: .LBB5_2:
@@ -2104,7 +2155,6 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, i64 %additive)
; GFX8-NEXT: v_readfirstlane_b32 s5, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s5
; GFX8-NEXT: v_mov_b32_e32 v1, s4
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mul_lo_u32 v3, s3, v2
; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s2, v2, v[0:1]
; GFX8-NEXT: s_mov_b32 s7, 0xf000
@@ -2118,23 +2168,23 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, i64 %additive)
; GFX9-LABEL: add_i64_uniform:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b64 s[6:7], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_cbranch_execz .LBB5_2
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_i32 s7, s3, s6
-; GFX9-NEXT: s_mul_hi_u32 s8, s2, s6
-; GFX9-NEXT: s_add_i32 s8, s8, s7
-; GFX9-NEXT: s_mul_i32 s6, s2, s6
-; GFX9-NEXT: v_mov_b32_e32 v0, s6
-; GFX9-NEXT: v_mov_b32_e32 v1, s8
+; GFX9-NEXT: s_mul_i32 s8, s2, s6
+; GFX9-NEXT: s_mul_hi_u32 s7, s2, s6
+; GFX9-NEXT: s_mul_i32 s6, s3, s6
+; GFX9-NEXT: s_add_u32 s9, s7, s6
; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v0, s8
+; GFX9-NEXT: v_mov_b32_e32 v1, s9
; GFX9-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: .LBB5_2:
@@ -2156,23 +2206,23 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, i64 %additive)
; GFX1064-LABEL: add_i64_uniform:
; GFX1064: ; %bb.0: ; %entry
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1064-NEXT: s_mov_b64 s[6:7], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], exec
+; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1064-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB5_2
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_mul_i32 s7, s3, s6
-; GFX1064-NEXT: s_mul_hi_u32 s8, s2, s6
+; GFX1064-NEXT: s_mul_hi_u32 s7, s2, s6
+; GFX1064-NEXT: s_mul_i32 s8, s3, s6
; GFX1064-NEXT: s_mul_i32 s6, s2, s6
-; GFX1064-NEXT: s_add_i32 s8, s8, s7
+; GFX1064-NEXT: s_add_u32 s7, s7, s8
+; GFX1064-NEXT: v_mov_b32_e32 v3, 0
; GFX1064-NEXT: v_mov_b32_e32 v0, s6
-; GFX1064-NEXT: v_mov_b32_e32 v1, s8
+; GFX1064-NEXT: v_mov_b32_e32 v1, s7
; GFX1064-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: buffer_gl0_inv
@@ -2192,21 +2242,21 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, i64 %additive)
; GFX1032-LABEL: add_i64_uniform:
; GFX1032: ; %bb.0: ; %entry
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1032-NEXT: s_mov_b32 s6, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s4, exec_lo
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s4
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB5_2
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s6
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_mul_i32 s6, s3, s5
; GFX1032-NEXT: s_mul_hi_u32 s7, s2, s5
-; GFX1032-NEXT: s_mul_i32 s5, s2, s5
-; GFX1032-NEXT: s_add_i32 s7, s7, s6
-; GFX1032-NEXT: v_mov_b32_e32 v0, s5
+; GFX1032-NEXT: s_mul_i32 s8, s3, s5
+; GFX1032-NEXT: s_mul_i32 s6, s2, s5
+; GFX1032-NEXT: s_add_u32 s7, s7, s8
+; GFX1032-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-NEXT: v_mov_b32_e32 v0, s6
; GFX1032-NEXT: v_mov_b32_e32 v1, s7
; GFX1032-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
@@ -2227,24 +2277,25 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, i64 %additive)
; GFX1164-LABEL: add_i64_uniform:
; GFX1164: ; %bb.0: ; %entry
; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1164-NEXT: s_mov_b64 s[6:7], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[4:5], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1164-NEXT: s_mov_b64 s[4:5], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1164-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB5_2
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1164-NEXT: v_mov_b32_e32 v3, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_mul_i32 s7, s3, s6
-; GFX1164-NEXT: s_mul_hi_u32 s8, s2, s6
+; GFX1164-NEXT: s_mul_hi_u32 s7, s2, s6
+; GFX1164-NEXT: s_mul_i32 s8, s3, s6
; GFX1164-NEXT: s_mul_i32 s6, s2, s6
-; GFX1164-NEXT: s_add_i32 s8, s8, s7
+; GFX1164-NEXT: s_add_u32 s7, s7, s8
+; GFX1164-NEXT: v_mov_b32_e32 v3, 0
; GFX1164-NEXT: v_mov_b32_e32 v0, s6
-; GFX1164-NEXT: v_mov_b32_e32 v1, s8
+; GFX1164-NEXT: v_mov_b32_e32 v1, s7
; GFX1164-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: buffer_gl0_inv
@@ -2265,23 +2316,22 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, i64 %additive)
; GFX1132-LABEL: add_i64_uniform:
; GFX1132: ; %bb.0: ; %entry
; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1132-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s4, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_bcnt1_i32_b32 s5, s4
+; GFX1132-NEXT: s_mov_b32 s4, exec_lo
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB5_2
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s5, s6
-; GFX1132-NEXT: v_mov_b32_e32 v3, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_mul_i32 s6, s3, s5
; GFX1132-NEXT: s_mul_hi_u32 s7, s2, s5
-; GFX1132-NEXT: s_mul_i32 s5, s2, s5
-; GFX1132-NEXT: s_add_i32 s7, s7, s6
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v1, s7
+; GFX1132-NEXT: s_mul_i32 s8, s3, s5
+; GFX1132-NEXT: s_mul_i32 s6, s2, s5
+; GFX1132-NEXT: s_add_u32 s7, s7, s8
+; GFX1132-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s6
+; GFX1132-NEXT: v_mov_b32_e32 v1, s7
; GFX1132-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: buffer_gl0_inv
@@ -2302,21 +2352,23 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, i64 %additive)
; GFX1364-LABEL: add_i64_uniform:
; GFX1364: ; %bb.0: ; %entry
; GFX1364-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1364-NEXT: s_mov_b64 s[6:7], exec
-; GFX1364-NEXT: s_mov_b32 s9, 0
-; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_mov_b64 s[4:5], exec
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1364-NEXT: s_mov_b64 s[4:5], exec
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1364-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1364-NEXT: s_cbranch_execz .LBB5_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: s_bcnt1_i32_b64 s8, s[6:7]
-; GFX1364-NEXT: v_mov_b32_e32 v3, 0
; GFX1364-NEXT: s_wait_kmcnt 0x0
-; GFX1364-NEXT: s_mul_u64 s[6:7], s[2:3], s[8:9]
-; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1364-NEXT: s_mul_hi_u32 s7, s2, s6
+; GFX1364-NEXT: s_mul_i32 s8, s3, s6
+; GFX1364-NEXT: s_mul_i32 s6, s2, s6
+; GFX1364-NEXT: s_add_co_u32 s7, s7, s8
+; GFX1364-NEXT: v_mov_b32_e32 v3, 0
; GFX1364-NEXT: v_mov_b32_e32 v0, s6
; GFX1364-NEXT: v_mov_b32_e32 v1, s7
; GFX1364-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
@@ -2338,26 +2390,27 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, i64 %additive)
; GFX1332-LABEL: add_i64_uniform:
; GFX1332: ; %bb.0: ; %entry
; GFX1332-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1332-NEXT: s_mov_b32 s6, exec_lo
-; GFX1332-NEXT: s_mov_b32 s5, 0
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
-; GFX1332-NEXT: s_mov_b32 s7, exec_lo
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1332-NEXT: s_mov_b32 s4, exec_lo
; GFX1332-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: s_bcnt1_i32_b32 s5, s4
+; GFX1332-NEXT: s_mov_b32 s4, exec_lo
; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1332-NEXT: s_cbranch_execz .LBB5_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: s_bcnt1_i32_b32 s4, s6
; GFX1332-NEXT: s_wait_kmcnt 0x0
-; GFX1332-NEXT: s_mul_u64 s[4:5], s[2:3], s[4:5]
-; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1332-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s4
-; GFX1332-NEXT: v_mov_b32_e32 v1, s5
+; GFX1332-NEXT: s_mul_hi_u32 s7, s2, s5
+; GFX1332-NEXT: s_mul_i32 s8, s3, s5
+; GFX1332-NEXT: s_mul_i32 s6, s2, s5
+; GFX1332-NEXT: s_add_co_u32 s7, s7, s8
+; GFX1332-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s6
+; GFX1332-NEXT: v_mov_b32_e32 v1, s7
; GFX1332-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
; GFX1332-NEXT: s_wait_dscnt 0x0
; GFX1332-NEXT: global_inv scope:SCOPE_SE
; GFX1332-NEXT: .LBB5_2:
-; GFX1332-NEXT: s_or_b32 exec_lo, exec_lo, s7
+; GFX1332-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX1332-NEXT: v_readfirstlane_b32 s5, v1
; GFX1332-NEXT: v_readfirstlane_b32 s4, v0
; GFX1332-NEXT: s_wait_kmcnt 0x0
@@ -3433,33 +3486,28 @@ entry:
define amdgpu_kernel void @add_i64_varying_nouse() {
; GFX7LESS_ITERATIVE-LABEL: add_i64_varying_nouse:
; GFX7LESS_ITERATIVE: ; %bb.0: ; %entry
+; GFX7LESS_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
-; GFX7LESS_ITERATIVE-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX7LESS_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
+; GFX7LESS_ITERATIVE-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
; GFX7LESS_ITERATIVE-NEXT: s_ff1_i32_b64 s4, s[2:3]
-; GFX7LESS_ITERATIVE-NEXT: s_nop 0
-; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s4
-; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s5, v1, s4
-; GFX7LESS_ITERATIVE-NEXT: s_add_u32 s0, s0, s6
-; GFX7LESS_ITERATIVE-NEXT: s_addc_u32 s1, s1, s5
-; GFX7LESS_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX7LESS_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[4:5]
-; GFX7LESS_ITERATIVE-NEXT: v_cmp_ne_u64_e64 s[4:5], s[2:3], 0
-; GFX7LESS_ITERATIVE-NEXT: s_and_b64 vcc, exec, s[4:5]
-; GFX7LESS_ITERATIVE-NEXT: s_cbranch_vccnz .LBB7_1
-; GFX7LESS_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s5, v0, s4
+; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s6, v2, s4
+; GFX7LESS_ITERATIVE-NEXT: s_add_u32 s0, s0, s5
+; GFX7LESS_ITERATIVE-NEXT: s_bitset0_b64 s[2:3], s4
+; GFX7LESS_ITERATIVE-NEXT: s_addc_u32 s1, s1, s6
+; GFX7LESS_ITERATIVE-NEXT: s_or_b64 s[4:5], s[2:3], s[2:3]
+; GFX7LESS_ITERATIVE-NEXT: s_cbranch_scc1 .LBB7_1
+; GFX7LESS_ITERATIVE-NEXT: ; %bb.2:
+; GFX7LESS_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS_ITERATIVE-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX7LESS_ITERATIVE-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX7LESS_ITERATIVE-NEXT: s_cbranch_execz .LBB7_4
; GFX7LESS_ITERATIVE-NEXT: ; %bb.3:
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v0, s0
; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v1, s1
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 m0, -1
; GFX7LESS_ITERATIVE-NEXT: ds_add_u64 v2, v[0:1]
; GFX7LESS_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
@@ -3468,30 +3516,28 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
;
; GFX8_ITERATIVE-LABEL: add_i64_varying_nouse:
; GFX8_ITERATIVE: ; %bb.0: ; %entry
+; GFX8_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX8_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
-; GFX8_ITERATIVE-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
+; GFX8_ITERATIVE-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
; GFX8_ITERATIVE-NEXT: s_ff1_i32_b64 s4, s[2:3]
-; GFX8_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s4
-; GFX8_ITERATIVE-NEXT: v_readlane_b32 s5, v1, s4
-; GFX8_ITERATIVE-NEXT: s_add_u32 s0, s0, s6
-; GFX8_ITERATIVE-NEXT: s_addc_u32 s1, s1, s5
-; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX8_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[4:5]
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s5, v0, s4
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s6, v2, s4
+; GFX8_ITERATIVE-NEXT: s_add_u32 s0, s0, s5
+; GFX8_ITERATIVE-NEXT: s_bitset0_b64 s[2:3], s4
+; GFX8_ITERATIVE-NEXT: s_addc_u32 s1, s1, s6
+; GFX8_ITERATIVE-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX8_ITERATIVE-NEXT: s_cbranch_scc1 .LBB7_1
-; GFX8_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX8_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX8_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX8_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8_ITERATIVE-NEXT: ; %bb.2:
+; GFX8_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX8_ITERATIVE-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX8_ITERATIVE-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX8_ITERATIVE-NEXT: s_cbranch_execz .LBB7_4
; GFX8_ITERATIVE-NEXT: ; %bb.3:
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v0, s0
; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v1, s1
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
; GFX8_ITERATIVE-NEXT: s_mov_b32 m0, -1
; GFX8_ITERATIVE-NEXT: ds_add_u64 v2, v[0:1]
; GFX8_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
@@ -3500,30 +3546,28 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
;
; GFX9_ITERATIVE-LABEL: add_i64_varying_nouse:
; GFX9_ITERATIVE: ; %bb.0: ; %entry
+; GFX9_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
-; GFX9_ITERATIVE-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
+; GFX9_ITERATIVE-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
; GFX9_ITERATIVE-NEXT: s_ff1_i32_b64 s4, s[2:3]
-; GFX9_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s4
-; GFX9_ITERATIVE-NEXT: v_readlane_b32 s5, v1, s4
-; GFX9_ITERATIVE-NEXT: s_add_u32 s0, s0, s6
-; GFX9_ITERATIVE-NEXT: s_addc_u32 s1, s1, s5
-; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX9_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[4:5]
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s5, v0, s4
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s6, v2, s4
+; GFX9_ITERATIVE-NEXT: s_add_u32 s0, s0, s5
+; GFX9_ITERATIVE-NEXT: s_bitset0_b64 s[2:3], s4
+; GFX9_ITERATIVE-NEXT: s_addc_u32 s1, s1, s6
+; GFX9_ITERATIVE-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX9_ITERATIVE-NEXT: s_cbranch_scc1 .LBB7_1
-; GFX9_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9_ITERATIVE-NEXT: ; %bb.2:
+; GFX9_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9_ITERATIVE-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX9_ITERATIVE-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX9_ITERATIVE-NEXT: s_cbranch_execz .LBB7_4
; GFX9_ITERATIVE-NEXT: ; %bb.3:
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v0, s0
; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v1, s1
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
; GFX9_ITERATIVE-NEXT: ds_add_u64 v2, v[0:1]
; GFX9_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX9_ITERATIVE-NEXT: .LBB7_4:
@@ -3531,30 +3575,28 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
;
; GFX1064_ITERATIVE-LABEL: add_i64_varying_nouse:
; GFX1064_ITERATIVE: ; %bb.0: ; %entry
-; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
-; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
-; GFX1064_ITERATIVE-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX1064_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064_ITERATIVE-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
; GFX1064_ITERATIVE-NEXT: s_ff1_i32_b64 s4, s[2:3]
; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s5, v0, s4
-; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s6, v1, s4
+; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s6, v2, s4
+; GFX1064_ITERATIVE-NEXT: s_bitset0_b64 s[2:3], s4
; GFX1064_ITERATIVE-NEXT: s_add_u32 s0, s0, s5
; GFX1064_ITERATIVE-NEXT: s_addc_u32 s1, s1, s6
-; GFX1064_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX1064_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[4:5]
+; GFX1064_ITERATIVE-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX1064_ITERATIVE-NEXT: s_cbranch_scc1 .LBB7_1
-; GFX1064_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064_ITERATIVE-NEXT: ; %bb.2:
+; GFX1064_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064_ITERATIVE-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX1064_ITERATIVE-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX1064_ITERATIVE-NEXT: s_cbranch_execz .LBB7_4
; GFX1064_ITERATIVE-NEXT: ; %bb.3:
-; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v0, s0
; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v1, s1
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
; GFX1064_ITERATIVE-NEXT: ds_add_u64 v2, v[0:1]
; GFX1064_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064_ITERATIVE-NEXT: buffer_gl0_inv
@@ -3563,29 +3605,27 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
;
; GFX1032_ITERATIVE-LABEL: add_i64_varying_nouse:
; GFX1032_ITERATIVE: ; %bb.0: ; %entry
-; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
-; GFX1032_ITERATIVE-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
; GFX1032_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
-; GFX1032_ITERATIVE-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX1032_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032_ITERATIVE-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032_ITERATIVE-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
; GFX1032_ITERATIVE-NEXT: s_ff1_i32_b32 s3, s2
; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s4, v0, s3
-; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s5, v2, s3
+; GFX1032_ITERATIVE-NEXT: s_bitset0_b32 s2, s3
; GFX1032_ITERATIVE-NEXT: s_add_u32 s0, s0, s4
; GFX1032_ITERATIVE-NEXT: s_addc_u32 s1, s1, s5
-; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
-; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s2, s2, s3
+; GFX1032_ITERATIVE-NEXT: s_cmp_lg_u32 s2, 0
; GFX1032_ITERATIVE-NEXT: s_cbranch_scc1 .LBB7_1
-; GFX1032_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032_ITERATIVE-NEXT: ; %bb.2:
+; GFX1032_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032_ITERATIVE-NEXT: s_and_saveexec_b32 s2, vcc_lo
-; GFX1032_ITERATIVE-NEXT: s_xor_b32 s2, exec_lo, s2
; GFX1032_ITERATIVE-NEXT: s_cbranch_execz .LBB7_4
; GFX1032_ITERATIVE-NEXT: ; %bb.3:
-; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v0, s0
; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
; GFX1032_ITERATIVE-NEXT: ds_add_u64 v2, v[0:1]
; GFX1032_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032_ITERATIVE-NEXT: buffer_gl0_inv
@@ -3594,34 +3634,31 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
;
; GFX1164_ITERATIVE-LABEL: add_i64_varying_nouse:
; GFX1164_ITERATIVE: ; %bb.0: ; %entry
-; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v1, 0x3ff, v0
; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
-; GFX1164_ITERATIVE-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX1164_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1164_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v2
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
+; GFX1164_ITERATIVE-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
; GFX1164_ITERATIVE-NEXT: s_ctz_i32_b64 s4, s[2:3]
-; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s5, v0, s4
-; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s6, v1, s4
+; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s5, v1, s4
+; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s6, v2, s4
+; GFX1164_ITERATIVE-NEXT: s_bitset0_b64 s[2:3], s4
; GFX1164_ITERATIVE-NEXT: s_add_u32 s0, s0, s5
; GFX1164_ITERATIVE-NEXT: s_addc_u32 s1, s1, s6
-; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164_ITERATIVE-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[4:5]
+; GFX1164_ITERATIVE-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX1164_ITERATIVE-NEXT: s_cbranch_scc1 .LBB7_1
-; GFX1164_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164_ITERATIVE-NEXT: ; %bb.2:
; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1164_ITERATIVE-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164_ITERATIVE-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX1164_ITERATIVE-NEXT: s_cbranch_execz .LBB7_4
; GFX1164_ITERATIVE-NEXT: ; %bb.3:
-; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v0, s0
; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v1, s1
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
; GFX1164_ITERATIVE-NEXT: ds_add_u64 v2, v[0:1]
; GFX1164_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164_ITERATIVE-NEXT: buffer_gl0_inv
@@ -3630,31 +3667,28 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
;
; GFX1132_ITERATIVE-LABEL: add_i64_varying_nouse:
; GFX1132_ITERATIVE: ; %bb.0: ; %entry
-; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX1132_ITERATIVE-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132_ITERATIVE-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1132_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
; GFX1132_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
-; GFX1132_ITERATIVE-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX1132_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132_ITERATIVE-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132_ITERATIVE-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1132_ITERATIVE-NEXT: s_ctz_i32_b32 s3, s2
-; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s4, v0, s3
-; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s5, v2, s3
+; GFX1132_ITERATIVE-NEXT: s_bitset0_b32 s2, s3
; GFX1132_ITERATIVE-NEXT: s_add_u32 s0, s0, s4
; GFX1132_ITERATIVE-NEXT: s_addc_u32 s1, s1, s5
-; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
-; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s2, s2, s3
+; GFX1132_ITERATIVE-NEXT: s_cmp_lg_u32 s2, 0
; GFX1132_ITERATIVE-NEXT: s_cbranch_scc1 .LBB7_1
-; GFX1132_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132_ITERATIVE-NEXT: ; %bb.2:
; GFX1132_ITERATIVE-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132_ITERATIVE-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132_ITERATIVE-NEXT: s_xor_b32 s2, exec_lo, s2
+; GFX1132_ITERATIVE-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132_ITERATIVE-NEXT: s_cbranch_execz .LBB7_4
; GFX1132_ITERATIVE-NEXT: ; %bb.3:
-; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s1
-; GFX1132_ITERATIVE-NEXT: v_mov_b32_e32 v0, s0
+; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX1132_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
; GFX1132_ITERATIVE-NEXT: ds_add_u64 v2, v[0:1]
; GFX1132_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132_ITERATIVE-NEXT: buffer_gl0_inv
@@ -3675,46 +3709,52 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
; GFX8_DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
; GFX8_DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v6
; GFX8_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX8_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
-; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX8_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1]
-; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX8_DPP-NEXT: v_add_u32_e32 v2, vcc, v2, v4
-; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX8_DPP-NEXT: v_add_u32_e32 v2, vcc, v2, v4
-; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX8_DPP-NEXT: v_add_u32_e32 v2, vcc, v2, v4
-; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX8_DPP-NEXT: v_add_u32_e32 v2, vcc, v2, v4
-; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX8_DPP-NEXT: v_add_u32_e32 v2, vcc, v2, v4
-; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX8_DPP-NEXT: v_add_u32_e32 v2, vcc, v2, v4
-; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; GFX8_DPP-NEXT: v_readlane_b32 s3, v1, 63
-; GFX8_DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX8_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]
+; GFX8_DPP-NEXT: v_cndmask_b32_e64 v2, 0, 0, s[0:1]
+; GFX8_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX8_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX8_DPP-NEXT: s_nop 0
+; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX8_DPP-NEXT: v_add_u32_e32 v1, vcc, v1, v3
+; GFX8_DPP-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc
+; GFX8_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX8_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX8_DPP-NEXT: s_nop 0
+; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX8_DPP-NEXT: v_add_u32_e32 v1, vcc, v1, v3
+; GFX8_DPP-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc
+; GFX8_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX8_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX8_DPP-NEXT: s_nop 0
+; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX8_DPP-NEXT: v_add_u32_e32 v1, vcc, v1, v3
+; GFX8_DPP-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc
+; GFX8_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX8_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX8_DPP-NEXT: s_nop 0
+; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX8_DPP-NEXT: v_add_u32_e32 v1, vcc, v1, v3
+; GFX8_DPP-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc
+; GFX8_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX8_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX8_DPP-NEXT: s_nop 0
+; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v4 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX8_DPP-NEXT: v_add_u32_e32 v1, vcc, v1, v3
+; GFX8_DPP-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc
+; GFX8_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX8_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX8_DPP-NEXT: s_nop 0
+; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v4 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX8_DPP-NEXT: v_add_u32_e32 v1, vcc, v1, v3
+; GFX8_DPP-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc
+; GFX8_DPP-NEXT: v_readlane_b32 s2, v1, 63
+; GFX8_DPP-NEXT: v_readlane_b32 s3, v2, 63
; GFX8_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX8_DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v6
@@ -3735,46 +3775,52 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
; GFX9_DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
; GFX9_DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v6
; GFX9_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
-; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX9_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1]
-; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9_DPP-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4
-; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
-; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9_DPP-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4
-; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
-; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9_DPP-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4
-; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
-; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9_DPP-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4
-; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
-; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9_DPP-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4
-; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
-; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9_DPP-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4
-; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
-; GFX9_DPP-NEXT: v_readlane_b32 s3, v1, 63
-; GFX9_DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX9_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]
+; GFX9_DPP-NEXT: v_cndmask_b32_e64 v2, 0, 0, s[0:1]
+; GFX9_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX9_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX9_DPP-NEXT: s_nop 0
+; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9_DPP-NEXT: v_add_co_u32_e32 v1, vcc, v1, v3
+; GFX9_DPP-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v4, vcc
+; GFX9_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX9_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX9_DPP-NEXT: s_nop 0
+; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9_DPP-NEXT: v_add_co_u32_e32 v1, vcc, v1, v3
+; GFX9_DPP-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v4, vcc
+; GFX9_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX9_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX9_DPP-NEXT: s_nop 0
+; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9_DPP-NEXT: v_add_co_u32_e32 v1, vcc, v1, v3
+; GFX9_DPP-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v4, vcc
+; GFX9_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX9_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX9_DPP-NEXT: s_nop 0
+; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9_DPP-NEXT: v_add_co_u32_e32 v1, vcc, v1, v3
+; GFX9_DPP-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v4, vcc
+; GFX9_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX9_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX9_DPP-NEXT: s_nop 0
+; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v4 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9_DPP-NEXT: v_add_co_u32_e32 v1, vcc, v1, v3
+; GFX9_DPP-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v4, vcc
+; GFX9_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX9_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX9_DPP-NEXT: s_nop 0
+; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v4 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9_DPP-NEXT: v_add_co_u32_e32 v1, vcc, v1, v3
+; GFX9_DPP-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v4, vcc
+; GFX9_DPP-NEXT: v_readlane_b32 s2, v1, 63
+; GFX9_DPP-NEXT: v_readlane_b32 s3, v2, 63
; GFX9_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9_DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v6
@@ -3791,55 +3837,64 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
; GFX1064_DPP-LABEL: add_i64_varying_nouse:
; GFX1064_DPP: ; %bb.0: ; %entry
; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064_DPP-NEXT: v_mov_b32_e32 v1, 0
-; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
-; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v3, 0, 0, s[0:1]
-; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, 0
-; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064_DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v2, v1
-; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v3, v4, vcc
-; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v6
-; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v5, vcc
-; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, 0
-; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v4
-; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v3, vcc
-; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v6
-; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v5, vcc
-; GFX1064_DPP-NEXT: v_permlanex16_b32 v3, v1, 0, 0
-; GFX1064_DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
+; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]
+; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v2, 0, 0, s[0:1]
+; GFX1064_DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064_DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v4, vcc
+; GFX1064_DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v4, vcc
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v4, vcc
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v3
; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v4, vcc
+; GFX1064_DPP-NEXT: ds_swizzle_b32 v3, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1064_DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(1)
+; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v4, vcc
+; GFX1064_DPP-NEXT: ds_permute_b32 v3, v5, v1
+; GFX1064_DPP-NEXT: ds_permute_b32 v4, v5, v2
+; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(1)
+; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v4, vcc
; GFX1064_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064_DPP-NEXT: v_readlane_b32 s2, v2, 0
-; GFX1064_DPP-NEXT: v_readlane_b32 s3, v1, 0
-; GFX1064_DPP-NEXT: v_readlane_b32 s4, v1, 32
-; GFX1064_DPP-NEXT: v_readlane_b32 s5, v2, 32
+; GFX1064_DPP-NEXT: v_readlane_b32 s2, v1, 63
+; GFX1064_DPP-NEXT: v_readlane_b32 s3, v2, 63
; GFX1064_DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
+; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1064_DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX1064_DPP-NEXT: s_add_u32 s0, s3, s4
-; GFX1064_DPP-NEXT: s_addc_u32 s1, s2, s5
-; GFX1064_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
+; GFX1064_DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1064_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v6
; GFX1064_DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064_DPP-NEXT: s_cbranch_execz .LBB7_2
; GFX1064_DPP-NEXT: ; %bb.1:
-; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, s1
-; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, s0
-; GFX1064_DPP-NEXT: ds_add_u64 v0, v[7:8]
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, s1
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, s0
+; GFX1064_DPP-NEXT: ds_add_u64 v0, v[6:7]
; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064_DPP-NEXT: buffer_gl0_inv
; GFX1064_DPP-NEXT: .LBB7_2:
@@ -3847,47 +3902,51 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
;
; GFX1032_DPP-LABEL: add_i64_varying_nouse:
; GFX1032_DPP: ; %bb.0: ; %entry
-; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v1, 0
-; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s0
-; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v3, 0, 0, s0
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, 0
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032_DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v2, v1
-; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v3, v4, vcc_lo
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v6
-; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v5, vcc_lo
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, 0
-; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v4
-; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v3, vcc_lo
-; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v6
-; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v5, vcc_lo
-; GFX1032_DPP-NEXT: v_permlanex16_b32 v3, v1, 0, 0
-; GFX1032_DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
+; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s2
+; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0, 0, s2
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v4, vcc_lo
-; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v9, exec_lo, 0
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
+; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v4, vcc_lo
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
+; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v4, vcc_lo
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
+; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v4, vcc_lo
+; GFX1032_DPP-NEXT: ds_swizzle_b32 v3, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1032_DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(1)
+; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
+; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v4, vcc_lo
+; GFX1032_DPP-NEXT: v_readlane_b32 s0, v1, 31
+; GFX1032_DPP-NEXT: v_readlane_b32 s1, v2, 31
+; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v5, exec_lo, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, v1
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, v2
-; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v9
-; GFX1032_DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v5
+; GFX1032_DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032_DPP-NEXT: s_cbranch_execz .LBB7_2
; GFX1032_DPP-NEXT: ; %bb.1:
-; GFX1032_DPP-NEXT: ds_add_u64 v0, v[7:8]
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, s1
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, s0
+; GFX1032_DPP-NEXT: ds_add_u64 v0, v[5:6]
; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032_DPP-NEXT: buffer_gl0_inv
; GFX1032_DPP-NEXT: .LBB7_2:
@@ -3898,60 +3957,85 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
; GFX1164_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
-; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1]
-; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
-; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
-; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
-; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]
+; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v2, 0, 0, s[0:1]
+; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1164_DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc
+; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
-; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
-; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v3, v3 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: v_permlanex16_b32 v3, v2, 0, 0
+; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc
+; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
-; GFX1164_DPP-NEXT: v_add_co_u32 v2, vcc, v2, v3
+; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc
+; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164_DPP-NEXT: v_permlanex16_b32 v4, v1, 0, 0
-; GFX1164_DPP-NEXT: v_permlane64_b32 v3, v2
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: ds_swizzle_b32 v3, v1 offset:swizzle(BROADCAST,32,15)
; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
-; GFX1164_DPP-NEXT: v_permlane64_b32 v4, v1
+; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc
+; GFX1164_DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(1)
+; GFX1164_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1164_DPP-NEXT: ds_permute_b32 v3, v5, v1
+; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(1)
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
+; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc
+; GFX1164_DPP-NEXT: ds_permute_b32 v4, v5, v2
+; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(1)
+; GFX1164_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
+; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: v_add_co_u32 v2, vcc, v2, v3
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0) depctr_va_vcc(0)
-; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v3, null, v1, v4, vcc
+; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 63
+; GFX1164_DPP-NEXT: v_readlane_b32 s3, v2, 63
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
+; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164_DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, v2
-; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, v3
-; GFX1164_DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164_DPP-NEXT: v_cmpx_eq_u32_e32 0, v7
+; GFX1164_DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1164_DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164_DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
; GFX1164_DPP-NEXT: s_cbranch_execz .LBB7_2
; GFX1164_DPP-NEXT: ; %bb.1:
-; GFX1164_DPP-NEXT: ds_add_u64 v0, v[5:6]
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, s1
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, s0
+; GFX1164_DPP-NEXT: ds_add_u64 v0, v[6:7]
; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164_DPP-NEXT: buffer_gl0_inv
; GFX1164_DPP-NEXT: .LBB7_2:
@@ -3960,47 +4044,58 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
; GFX1132_DPP-LABEL: add_i64_varying_nouse:
; GFX1132_DPP: ; %bb.0: ; %entry
; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s0
-; GFX1132_DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s0
-; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1132_DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc_lo
-; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1132_DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc_lo
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132_DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc_lo
-; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc_lo, v3, v3 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s2
+; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v2, 0, 0, s2
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132_DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v4, v2
+; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132_DPP-NEXT: v_permlanex16_b32 v3, v2, 0, 0
+; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo
+; GFX1132_DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v4, v2
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
+; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132_DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v4, v2
+; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc_lo
-; GFX1132_DPP-NEXT: v_add_co_u32 v2, vcc_lo, v2, v3
+; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo
+; GFX1132_DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v4, v2
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132_DPP-NEXT: v_permlanex16_b32 v4, v1, 0, 0
-; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v3, null, v1, v4, vcc_lo
-; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1132_DPP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v5, v2
-; GFX1132_DPP-NEXT: v_mov_b32_e32 v6, v3
-; GFX1132_DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132_DPP-NEXT: v_cmpx_eq_u32_e32 0, v7
+; GFX1132_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
+; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo
+; GFX1132_DPP-NEXT: ds_swizzle_b32 v3, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132_DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(1)
+; GFX1132_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
+; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132_DPP-NEXT: v_readlane_b32 s0, v1, 31
+; GFX1132_DPP-NEXT: v_readlane_b32 s1, v2, 31
+; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v5, exec_lo, 0
+; GFX1132_DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1132_DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132_DPP-NEXT: v_cmpx_eq_u32_e32 0, v5
; GFX1132_DPP-NEXT: s_cbranch_execz .LBB7_2
; GFX1132_DPP-NEXT: ; %bb.1:
+; GFX1132_DPP-NEXT: v_dual_mov_b32 v6, s1 :: v_dual_mov_b32 v5, s0
; GFX1132_DPP-NEXT: ds_add_u64 v0, v[5:6]
; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132_DPP-NEXT: buffer_gl0_inv
@@ -4011,57 +4106,81 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
; GFX1364: ; %bb.0: ; %entry
; GFX1364-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1364-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1364-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1]
-; GFX1364-NEXT: v_mov_b32_e32 v2, 0
-; GFX1364-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
-; GFX1364-NEXT: v_mov_b32_e32 v4, 0
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1364-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]
+; GFX1364-NEXT: v_cndmask_b32_e64 v2, 0, 0, s[0:1]
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1364-NEXT: v_mov_b32_dpp v2, v1 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1364-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1364-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
-; GFX1364-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1364-NEXT: v_mov_b32_e32 v2, 0
+; GFX1364-NEXT: v_mov_b32_e32 v3, v1
+; GFX1364-NEXT: v_mov_b32_e32 v4, v2
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1364-NEXT: v_mov_b32_dpp v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1364-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1364-NEXT: v_add_nc_u32_e32 v5, 32, v5
; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mov_b32_dpp v4, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1364-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1364-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1364-NEXT: v_mov_b32_e32 v4, 0
-; GFX1364-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1364-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
-; GFX1364-NEXT: v_add_co_u32_e64_dpp v2, vcc, v3, v3 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1364-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1364-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1364-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1364-NEXT: v_mov_b32_e32 v3, v1
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1364-NEXT: v_mov_b32_e32 v4, v2
+; GFX1364-NEXT: v_mov_b32_dpp v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1364-NEXT: v_mov_b32_dpp v4, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1364-NEXT: v_permlanex16_b32 v3, v2, 0, 0
+; GFX1364-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1364-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1364-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc
+; GFX1364-NEXT: v_mov_b32_e32 v3, v1
; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1364-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
-; GFX1364-NEXT: v_add_co_u32 v2, vcc, v2, v3
+; GFX1364-NEXT: v_mov_b32_e32 v4, v2
+; GFX1364-NEXT: v_mov_b32_dpp v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1364-NEXT: v_permlanex16_b32 v4, v1, 0, 0
-; GFX1364-NEXT: v_permlane64_b32 v3, v2
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
-; GFX1364-NEXT: v_permlane64_b32 v4, v1
+; GFX1364-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1364-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1364-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc
+; GFX1364-NEXT: v_mov_b32_e32 v3, v1
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1364-NEXT: v_mov_b32_e32 v4, v2
+; GFX1364-NEXT: v_mov_b32_dpp v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1364-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1364-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1364-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc
+; GFX1364-NEXT: ds_swizzle_b32 v3, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1364-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1364-NEXT: s_wait_dscnt 0x1
+; GFX1364-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1364-NEXT: s_wait_dscnt 0x0
+; GFX1364-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc
+; GFX1364-NEXT: ds_permute_b32 v3, v5, v1
+; GFX1364-NEXT: ds_permute_b32 v4, v5, v2
+; GFX1364-NEXT: s_wait_dscnt 0x1
+; GFX1364-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1364-NEXT: s_wait_dscnt 0x0
+; GFX1364-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc
; GFX1364-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1364-NEXT: v_add_co_u32 v2, vcc, v2, v3
-; GFX1364-NEXT: v_add_co_ci_u32_e64 v3, null, v1, v4, vcc
+; GFX1364-NEXT: v_readlane_b32 s2, v1, 63
+; GFX1364-NEXT: v_readlane_b32 s3, v2, 63
; GFX1364-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1364-NEXT: v_mov_b32_e32 v0, 0
-; GFX1364-NEXT: v_mov_b32_e32 v5, v2
-; GFX1364-NEXT: v_mov_b32_e32 v6, v3
-; GFX1364-NEXT: s_mov_b64 s[0:1], exec
-; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v7
+; GFX1364-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1364-NEXT: s_mov_b64 s[2:3], exec
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v6
; GFX1364-NEXT: s_cbranch_execz .LBB7_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: ds_add_u64 v0, v[5:6]
+; GFX1364-NEXT: v_mov_b32_e32 v7, s1
+; GFX1364-NEXT: v_mov_b32_e32 v6, s0
+; GFX1364-NEXT: ds_add_u64 v0, v[6:7]
; GFX1364-NEXT: s_wait_dscnt 0x0
; GFX1364-NEXT: global_inv scope:SCOPE_SE
; GFX1364-NEXT: .LBB7_2:
@@ -4070,46 +4189,58 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
; GFX1332-LABEL: add_i64_varying_nouse:
; GFX1332: ; %bb.0: ; %entry
; GFX1332-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1332-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1332-NEXT: v_cndmask_b32_e64 v1, 0, 0, s0
-; GFX1332-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_cndmask_b32 v3, 0, v0, s0
-; GFX1332-NEXT: v_mov_b32_e32 v4, 0
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1332-NEXT: v_mov_b32_dpp v2, v1 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1332-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1332-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc_lo
-; GFX1332-NEXT: v_mov_b32_e32 v2, 0
-; GFX1332-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-NEXT: v_mov_b32_dpp v4, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1332-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc_lo
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1332-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1332-NEXT: v_mov_b32_e32 v4, 0
-; GFX1332-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1332-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc_lo
-; GFX1332-NEXT: v_add_co_u32_e64_dpp v2, vcc_lo, v3, v3 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1332-NEXT: v_mov_b32_dpp v4, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1332-NEXT: v_permlanex16_b32 v3, v2, 0, 0
+; GFX1332-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: v_cndmask_b32_e64 v2, 0, 0, s2
+; GFX1332-NEXT: v_dual_cndmask_b32 v1, 0, v0, s2 :: v_dual_mov_b32 v4, v2
+; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1332-NEXT: v_mov_b32_e32 v3, v1
+; GFX1332-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: v_mov_b32_dpp v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1332-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
+; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo
+; GFX1332-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v4, v2
+; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1332-NEXT: v_mov_b32_dpp v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1332-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
+; GFX1332-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo
+; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v4, v2
+; GFX1332-NEXT: v_mov_b32_dpp v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1332-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc_lo
-; GFX1332-NEXT: v_add_co_u32 v2, vcc_lo, v2, v3
+; GFX1332-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1332-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
+; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo
+; GFX1332-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v4, v2
+; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1332-NEXT: v_mov_b32_dpp v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1332-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-NEXT: v_permlanex16_b32 v4, v1, 0, 0
-; GFX1332-NEXT: v_add_co_ci_u32_e64 v3, null, v1, v4, vcc_lo
-; GFX1332-NEXT: s_mov_b32 exec_lo, s0
-; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1332-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v5, v2
-; GFX1332-NEXT: v_mov_b32_e32 v6, v3
-; GFX1332-NEXT: s_mov_b32 s0, exec_lo
-; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v7
+; GFX1332-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
+; GFX1332-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo
+; GFX1332-NEXT: ds_swizzle_b32 v3, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1332-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1332-NEXT: s_wait_dscnt 0x1
+; GFX1332-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
+; GFX1332-NEXT: s_wait_dscnt 0x0
+; GFX1332-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo
+; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1332-NEXT: v_readlane_b32 s0, v1, 31
+; GFX1332-NEXT: v_readlane_b32 s1, v2, 31
+; GFX1332-NEXT: s_mov_b32 exec_lo, s2
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v5, exec_lo, 0
+; GFX1332-NEXT: v_mov_b32_e32 v0, 0
+; GFX1332-NEXT: s_mov_b32 s2, exec_lo
+; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v5
; GFX1332-NEXT: s_cbranch_execz .LBB7_2
; GFX1332-NEXT: ; %bb.1:
+; GFX1332-NEXT: v_dual_mov_b32 v6, s1 :: v_dual_mov_b32 v5, s0
; GFX1332-NEXT: ds_add_u64 v0, v[5:6]
; GFX1332-NEXT: s_wait_dscnt 0x0
; GFX1332-NEXT: global_inv scope:SCOPE_SE
@@ -4125,15 +4256,15 @@ entry:
define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out) {
; GFX7LESS-LABEL: sub_i32_constant:
; GFX7LESS: ; %bb.0: ; %entry
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX7LESS-NEXT: ; implicit-def: $vgpr1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB8_2
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX7LESS-NEXT: s_mul_i32 s2, s2, 5
; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0
; GFX7LESS-NEXT: v_mov_b32_e32 v2, s2
@@ -4154,15 +4285,15 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out) {
;
; GFX8-LABEL: sub_i32_constant:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB8_2
; GFX8-NEXT: ; %bb.1:
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX8-NEXT: s_mul_i32 s2, s2, 5
; GFX8-NEXT: v_mov_b32_e32 v1, 0
; GFX8-NEXT: v_mov_b32_e32 v2, s2
@@ -4183,15 +4314,15 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out) {
;
; GFX9-LABEL: sub_i32_constant:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB8_2
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_mul_i32 s2, s2, 5
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: v_mov_b32_e32 v2, s2
@@ -4211,17 +4342,17 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out) {
;
; GFX1064-LABEL: sub_i32_constant:
; GFX1064: ; %bb.0: ; %entry
-; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: ; implicit-def: $vgpr1
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB8_2
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-NEXT: s_mul_i32 s2, s2, 5
+; GFX1064-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-NEXT: v_mov_b32_e32 v2, s2
; GFX1064-NEXT: ds_sub_rtn_u32 v1, v1, v2
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
@@ -4241,16 +4372,16 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out) {
;
; GFX1032-LABEL: sub_i32_constant:
; GFX1032: ; %bb.0: ; %entry
-; GFX1032-NEXT: s_mov_b32 s1, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, exec_lo
; GFX1032-NEXT: ; implicit-def: $vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s1, s0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB8_2
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-NEXT: s_mul_i32 s1, s1, 5
+; GFX1032-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-NEXT: v_mov_b32_e32 v2, s1
; GFX1032-NEXT: ds_sub_rtn_u32 v1, v1, v2
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
@@ -4270,19 +4401,19 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out) {
;
; GFX1164-LABEL: sub_i32_constant:
; GFX1164: ; %bb.0: ; %entry
-; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX1164-NEXT: ; implicit-def: $vgpr1
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-NEXT: s_cbranch_execz .LBB8_2
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0
; GFX1164-NEXT: s_mul_i32 s2, s2, 5
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT: v_mov_b32_e32 v1, 0
; GFX1164-NEXT: v_mov_b32_e32 v2, s2
; GFX1164-NEXT: ds_sub_rtn_u32 v1, v1, v2
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
@@ -4302,17 +4433,17 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out) {
;
; GFX1132-LABEL: sub_i32_constant:
; GFX1132: ; %bb.0: ; %entry
-; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX1132-NEXT: ; implicit-def: $vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX1132-NEXT: s_mov_b32 s0, exec_lo
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB8_2
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1132-NEXT: s_mul_i32 s1, s1, 5
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s1
; GFX1132-NEXT: ds_sub_rtn_u32 v1, v1, v2
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
@@ -4332,19 +4463,19 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out) {
;
; GFX1364-LABEL: sub_i32_constant:
; GFX1364: ; %bb.0: ; %entry
-; GFX1364-NEXT: s_mov_b64 s[2:3], exec
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_mov_b64 s[0:1], exec
-; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX1364-NEXT: ; implicit-def: $vgpr1
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX1364-NEXT: s_mov_b64 s[0:1], exec
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1364-NEXT: s_cbranch_execz .LBB8_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1364-NEXT: v_mov_b32_e32 v1, 0
; GFX1364-NEXT: s_mul_i32 s2, s2, 5
-; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1364-NEXT: v_mov_b32_e32 v1, 0
; GFX1364-NEXT: v_mov_b32_e32 v2, s2
; GFX1364-NEXT: ds_sub_rtn_u32 v1, v1, v2
; GFX1364-NEXT: s_wait_dscnt 0x0
@@ -4364,17 +4495,17 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out) {
;
; GFX1332-LABEL: sub_i32_constant:
; GFX1332: ; %bb.0: ; %entry
-; GFX1332-NEXT: s_mov_b32 s1, exec_lo
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1332-NEXT: s_mov_b32 s0, exec_lo
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX1332-NEXT: ; implicit-def: $vgpr1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX1332-NEXT: s_mov_b32 s0, exec_lo
; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1332-NEXT: s_cbranch_execz .LBB8_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1332-NEXT: s_mul_i32 s1, s1, 5
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1332-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s1
; GFX1332-NEXT: ds_sub_rtn_u32 v1, v1, v2
; GFX1332-NEXT: s_wait_dscnt 0x0
@@ -4400,20 +4531,20 @@ entry:
define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, i32 %subitive) {
; GFX7LESS-LABEL: sub_i32_uniform:
; GFX7LESS: ; %bb.0: ; %entry
-; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-NEXT: s_load_dword s2, s[4:5], 0xb
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX7LESS-NEXT: ; implicit-def: $vgpr1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB9_2
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_mul_i32 s2, s6, s2
+; GFX7LESS-NEXT: s_mul_i32 s3, s2, s3
; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, s2
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s3
; GFX7LESS-NEXT: s_mov_b32 m0, -1
; GFX7LESS-NEXT: ds_sub_rtn_u32 v1, v1, v2
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
@@ -4421,7 +4552,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, i32 %subitive)
; GFX7LESS-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX7LESS-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v1
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
@@ -4431,20 +4562,20 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, i32 %subitive)
;
; GFX8-LABEL: sub_i32_uniform:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB9_2
; GFX8-NEXT: ; %bb.1:
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mul_i32 s2, s6, s2
+; GFX8-NEXT: s_mul_i32 s3, s2, s3
; GFX8-NEXT: v_mov_b32_e32 v1, 0
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: v_mov_b32_e32 v2, s3
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_sub_rtn_u32 v1, v1, v2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
@@ -4452,7 +4583,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, i32 %subitive)
; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX8-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX8-NEXT: v_readfirstlane_b32 s4, v1
; GFX8-NEXT: s_mov_b32 s3, 0xf000
; GFX8-NEXT: s_mov_b32 s2, -1
@@ -4462,27 +4593,27 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, i32 %subitive)
;
; GFX9-LABEL: sub_i32_uniform:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB9_2
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_i32 s2, s6, s2
+; GFX9-NEXT: s_mul_i32 s3, s2, s3
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_mov_b32_e32 v2, s2
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
; GFX9-NEXT: ds_sub_rtn_u32 v1, v1, v2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: .LBB9_2:
; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX9-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX9-NEXT: v_readfirstlane_b32 s4, v1
; GFX9-NEXT: s_mov_b32 s3, 0xf000
; GFX9-NEXT: s_mov_b32 s2, -1
@@ -4492,20 +4623,20 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, i32 %subitive)
;
; GFX1064-LABEL: sub_i32_uniform:
; GFX1064: ; %bb.0: ; %entry
-; GFX1064-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: ; implicit-def: $vgpr1
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB9_2
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_mul_i32 s2, s6, s2
-; GFX1064-NEXT: v_mov_b32_e32 v2, s2
+; GFX1064-NEXT: s_mul_i32 s3, s2, s3
+; GFX1064-NEXT: v_mov_b32_e32 v1, 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, s3
; GFX1064-NEXT: ds_sub_rtn_u32 v1, v1, v2
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: buffer_gl0_inv
@@ -4514,7 +4645,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, i32 %subitive)
; GFX1064-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX1064-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX1064-NEXT: v_readfirstlane_b32 s2, v1
; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
; GFX1064-NEXT: v_sub_nc_u32_e32 v0, s2, v0
@@ -4525,17 +4656,17 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, i32 %subitive)
; GFX1032-LABEL: sub_i32_uniform:
; GFX1032: ; %bb.0: ; %entry
; GFX1032-NEXT: s_load_dword s0, s[4:5], 0x2c
-; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s1, exec_lo
; GFX1032-NEXT: ; implicit-def: $vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s2, s1
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB9_2
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s2, s2
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_mul_i32 s2, s0, s2
+; GFX1032-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-NEXT: v_mov_b32_e32 v2, s2
; GFX1032-NEXT: ds_sub_rtn_u32 v1, v1, v2
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
@@ -4555,22 +4686,22 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, i32 %subitive)
;
; GFX1164-LABEL: sub_i32_uniform:
; GFX1164: ; %bb.0: ; %entry
-; GFX1164-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX1164-NEXT: ; implicit-def: $vgpr1
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-NEXT: s_cbranch_execz .LBB9_2
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_mul_i32 s2, s6, s2
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: v_mov_b32_e32 v2, s2
+; GFX1164-NEXT: s_mul_i32 s3, s2, s3
+; GFX1164-NEXT: v_mov_b32_e32 v1, 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, s3
; GFX1164-NEXT: ds_sub_rtn_u32 v1, v1, v2
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: buffer_gl0_inv
@@ -4578,7 +4709,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, i32 %subitive)
; GFX1164-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX1164-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX1164-NEXT: v_readfirstlane_b32 s2, v1
; GFX1164-NEXT: s_mov_b32 s3, 0x31016000
; GFX1164-NEXT: v_sub_nc_u32_e32 v0, s2, v0
@@ -4589,15 +4720,15 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, i32 %subitive)
; GFX1132-LABEL: sub_i32_uniform:
; GFX1132: ; %bb.0: ; %entry
; GFX1132-NEXT: s_load_b32 s0, s[4:5], 0x2c
-; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX1132-NEXT: ; implicit-def: $vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB9_2
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_mul_i32 s2, s0, s2
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -4619,22 +4750,22 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, i32 %subitive)
;
; GFX1364-LABEL: sub_i32_uniform:
; GFX1364: ; %bb.0: ; %entry
-; GFX1364-NEXT: s_load_b32 s6, s[4:5], 0x2c nv
-; GFX1364-NEXT: s_mov_b64 s[2:3], exec
+; GFX1364-NEXT: s_load_b32 s2, s[4:5], 0x2c nv
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_mov_b64 s[0:1], exec
-; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX1364-NEXT: ; implicit-def: $vgpr1
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX1364-NEXT: s_mov_b64 s[0:1], exec
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1364-NEXT: s_cbranch_execz .LBB9_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1364-NEXT: v_mov_b32_e32 v1, 0
; GFX1364-NEXT: s_wait_kmcnt 0x0
-; GFX1364-NEXT: s_mul_i32 s2, s6, s2
-; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1364-NEXT: v_mov_b32_e32 v2, s2
+; GFX1364-NEXT: s_mul_i32 s3, s2, s3
+; GFX1364-NEXT: v_mov_b32_e32 v1, 0
+; GFX1364-NEXT: v_mov_b32_e32 v2, s3
; GFX1364-NEXT: ds_sub_rtn_u32 v1, v1, v2
; GFX1364-NEXT: s_wait_dscnt 0x0
; GFX1364-NEXT: global_inv scope:SCOPE_SE
@@ -4642,7 +4773,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, i32 %subitive)
; GFX1364-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX1364-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1364-NEXT: s_wait_kmcnt 0x0
-; GFX1364-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX1364-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX1364-NEXT: v_readfirstlane_b32 s2, v1
; GFX1364-NEXT: s_mov_b32 s3, 0x31016000
; GFX1364-NEXT: v_sub_nc_u32_e32 v0, s2, v0
@@ -4653,15 +4784,15 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, i32 %subitive)
; GFX1332-LABEL: sub_i32_uniform:
; GFX1332: ; %bb.0: ; %entry
; GFX1332-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
-; GFX1332-NEXT: s_mov_b32 s2, exec_lo
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1332-NEXT: s_mov_b32 s1, exec_lo
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX1332-NEXT: ; implicit-def: $vgpr1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX1332-NEXT: s_mov_b32 s1, exec_lo
; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1332-NEXT: s_cbranch_execz .LBB9_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX1332-NEXT: s_wait_kmcnt 0x0
; GFX1332-NEXT: s_mul_i32 s2, s0, s2
; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -5396,112 +5527,102 @@ entry:
define amdgpu_kernel void @sub_i32_varying_nouse() {
; GFX7LESS_ITERATIVE-LABEL: sub_i32_varying_nouse:
; GFX7LESS_ITERATIVE: ; %bb.0: ; %entry
-; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s2, 0
-; GFX7LESS_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX7LESS_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS_ITERATIVE-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
; GFX7LESS_ITERATIVE-NEXT: s_ff1_i32_b64 s3, s[0:1]
-; GFX7LESS_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s3
-; GFX7LESS_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS_ITERATIVE-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
-; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s3
-; GFX7LESS_ITERATIVE-NEXT: s_add_i32 s2, s2, s6
-; GFX7LESS_ITERATIVE-NEXT: s_and_b64 vcc, exec, s[4:5]
-; GFX7LESS_ITERATIVE-NEXT: s_cbranch_vccnz .LBB11_1
-; GFX7LESS_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s4, v0, s3
+; GFX7LESS_ITERATIVE-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX7LESS_ITERATIVE-NEXT: s_add_i32 s2, s2, s4
+; GFX7LESS_ITERATIVE-NEXT: s_or_b64 s[4:5], s[0:1], s[0:1]
+; GFX7LESS_ITERATIVE-NEXT: s_cbranch_scc1 .LBB11_1
+; GFX7LESS_ITERATIVE-NEXT: ; %bb.2:
+; GFX7LESS_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS_ITERATIVE-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS_ITERATIVE-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS_ITERATIVE-NEXT: s_cbranch_execz .LBB11_4
; GFX7LESS_ITERATIVE-NEXT: ; %bb.3:
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v1, s2
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v0, s2
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 m0, -1
-; GFX7LESS_ITERATIVE-NEXT: ds_sub_u32 v0, v1
+; GFX7LESS_ITERATIVE-NEXT: ds_sub_u32 v1, v0
; GFX7LESS_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS_ITERATIVE-NEXT: .LBB11_4:
; GFX7LESS_ITERATIVE-NEXT: s_endpgm
;
; GFX8_ITERATIVE-LABEL: sub_i32_varying_nouse:
; GFX8_ITERATIVE: ; %bb.0: ; %entry
-; GFX8_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX8_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX8_ITERATIVE-NEXT: s_mov_b32 s2, 0
-; GFX8_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX8_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX8_ITERATIVE-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
; GFX8_ITERATIVE-NEXT: s_ff1_i32_b64 s3, s[0:1]
; GFX8_ITERATIVE-NEXT: v_readlane_b32 s4, v0, s3
+; GFX8_ITERATIVE-NEXT: s_bitset0_b64 s[0:1], s3
; GFX8_ITERATIVE-NEXT: s_add_i32 s2, s2, s4
-; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s3
-; GFX8_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
+; GFX8_ITERATIVE-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX8_ITERATIVE-NEXT: s_cbranch_scc1 .LBB11_1
-; GFX8_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX8_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX8_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX8_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8_ITERATIVE-NEXT: ; %bb.2:
+; GFX8_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX8_ITERATIVE-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX8_ITERATIVE-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX8_ITERATIVE-NEXT: s_cbranch_execz .LBB11_4
; GFX8_ITERATIVE-NEXT: ; %bb.3:
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v0, 0
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v1, s2
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v0, s2
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
; GFX8_ITERATIVE-NEXT: s_mov_b32 m0, -1
-; GFX8_ITERATIVE-NEXT: ds_sub_u32 v0, v1
+; GFX8_ITERATIVE-NEXT: ds_sub_u32 v1, v0
; GFX8_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX8_ITERATIVE-NEXT: .LBB11_4:
; GFX8_ITERATIVE-NEXT: s_endpgm
;
; GFX9_ITERATIVE-LABEL: sub_i32_varying_nouse:
; GFX9_ITERATIVE: ; %bb.0: ; %entry
-; GFX9_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX9_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX9_ITERATIVE-NEXT: s_mov_b32 s2, 0
-; GFX9_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX9_ITERATIVE-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
; GFX9_ITERATIVE-NEXT: s_ff1_i32_b64 s3, s[0:1]
; GFX9_ITERATIVE-NEXT: v_readlane_b32 s4, v0, s3
+; GFX9_ITERATIVE-NEXT: s_bitset0_b64 s[0:1], s3
; GFX9_ITERATIVE-NEXT: s_add_i32 s2, s2, s4
-; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s3
-; GFX9_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
+; GFX9_ITERATIVE-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX9_ITERATIVE-NEXT: s_cbranch_scc1 .LBB11_1
-; GFX9_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9_ITERATIVE-NEXT: ; %bb.2:
+; GFX9_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9_ITERATIVE-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9_ITERATIVE-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9_ITERATIVE-NEXT: s_cbranch_execz .LBB11_4
; GFX9_ITERATIVE-NEXT: ; %bb.3:
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v0, 0
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v1, s2
-; GFX9_ITERATIVE-NEXT: ds_sub_u32 v0, v1
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v0, s2
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
+; GFX9_ITERATIVE-NEXT: ds_sub_u32 v1, v0
; GFX9_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX9_ITERATIVE-NEXT: .LBB11_4:
; GFX9_ITERATIVE-NEXT: s_endpgm
;
; GFX1064_ITERATIVE-LABEL: sub_i32_varying_nouse:
; GFX1064_ITERATIVE: ; %bb.0: ; %entry
-; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064_ITERATIVE-NEXT: s_mov_b32 s2, 0
-; GFX1064_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX1064_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064_ITERATIVE-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
; GFX1064_ITERATIVE-NEXT: s_ff1_i32_b64 s3, s[0:1]
-; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1064_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s3
-; GFX1064_ITERATIVE-NEXT: s_add_i32 s2, s2, s6
-; GFX1064_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
+; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1064_ITERATIVE-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1064_ITERATIVE-NEXT: s_add_i32 s2, s2, s4
+; GFX1064_ITERATIVE-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX1064_ITERATIVE-NEXT: s_cbranch_scc1 .LBB11_1
-; GFX1064_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064_ITERATIVE-NEXT: ; %bb.2:
+; GFX1064_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064_ITERATIVE-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064_ITERATIVE-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064_ITERATIVE-NEXT: s_cbranch_execz .LBB11_4
; GFX1064_ITERATIVE-NEXT: ; %bb.3:
-; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v0, 0
-; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064_ITERATIVE-NEXT: ds_sub_u32 v0, v1
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v0, s2
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
+; GFX1064_ITERATIVE-NEXT: ds_sub_u32 v1, v0
; GFX1064_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064_ITERATIVE-NEXT: buffer_gl0_inv
; GFX1064_ITERATIVE-NEXT: .LBB11_4:
@@ -5509,26 +5630,24 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
;
; GFX1032_ITERATIVE-LABEL: sub_i32_varying_nouse:
; GFX1032_ITERATIVE: ; %bb.0: ; %entry
-; GFX1032_ITERATIVE-NEXT: s_mov_b32 s1, exec_lo
+; GFX1032_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1032_ITERATIVE-NEXT: s_mov_b32 s0, 0
-; GFX1032_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX1032_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032_ITERATIVE-NEXT: s_mov_b32 s1, exec_lo
+; GFX1032_ITERATIVE-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
; GFX1032_ITERATIVE-NEXT: s_ff1_i32_b32 s2, s1
; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s2, 1, s2
+; GFX1032_ITERATIVE-NEXT: s_bitset0_b32 s1, s2
; GFX1032_ITERATIVE-NEXT: s_add_i32 s0, s0, s3
-; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s1, s1, s2
+; GFX1032_ITERATIVE-NEXT: s_cmp_lg_u32 s1, 0
; GFX1032_ITERATIVE-NEXT: s_cbranch_scc1 .LBB11_1
-; GFX1032_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032_ITERATIVE-NEXT: ; %bb.2:
+; GFX1032_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032_ITERATIVE-NEXT: s_and_saveexec_b32 s1, vcc_lo
-; GFX1032_ITERATIVE-NEXT: s_xor_b32 s1, exec_lo, s1
; GFX1032_ITERATIVE-NEXT: s_cbranch_execz .LBB11_4
; GFX1032_ITERATIVE-NEXT: ; %bb.3:
-; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v0, 0
-; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v1, s0
-; GFX1032_ITERATIVE-NEXT: ds_sub_u32 v0, v1
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
+; GFX1032_ITERATIVE-NEXT: ds_sub_u32 v1, v0
; GFX1032_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032_ITERATIVE-NEXT: buffer_gl0_inv
; GFX1032_ITERATIVE-NEXT: .LBB11_4:
@@ -5536,30 +5655,28 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
;
; GFX1164_ITERATIVE-LABEL: sub_i32_varying_nouse:
; GFX1164_ITERATIVE: ; %bb.0: ; %entry
+; GFX1164_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
; GFX1164_ITERATIVE-NEXT: s_mov_b32 s2, 0
-; GFX1164_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX1164_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1164_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164_ITERATIVE-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
; GFX1164_ITERATIVE-NEXT: s_ctz_i32_b64 s3, s[0:1]
-; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s3
-; GFX1164_ITERATIVE-NEXT: s_add_i32 s2, s2, s6
-; GFX1164_ITERATIVE-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[4:5]
+; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1164_ITERATIVE-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164_ITERATIVE-NEXT: s_add_i32 s2, s2, s4
+; GFX1164_ITERATIVE-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX1164_ITERATIVE-NEXT: s_cbranch_scc1 .LBB11_1
-; GFX1164_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164_ITERATIVE-NEXT: ; %bb.2:
; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164_ITERATIVE-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164_ITERATIVE-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164_ITERATIVE-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164_ITERATIVE-NEXT: s_cbranch_execz .LBB11_4
; GFX1164_ITERATIVE-NEXT: ; %bb.3:
-; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v0, 0
-; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v1, s2
-; GFX1164_ITERATIVE-NEXT: ds_sub_u32 v0, v1
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v0, s2
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
+; GFX1164_ITERATIVE-NEXT: ds_sub_u32 v1, v0
; GFX1164_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164_ITERATIVE-NEXT: buffer_gl0_inv
; GFX1164_ITERATIVE-NEXT: .LBB11_4:
@@ -5567,28 +5684,26 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
;
; GFX1132_ITERATIVE-LABEL: sub_i32_varying_nouse:
; GFX1132_ITERATIVE: ; %bb.0: ; %entry
+; GFX1132_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1132_ITERATIVE-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1132_ITERATIVE-NEXT: s_mov_b32 s1, exec_lo
; GFX1132_ITERATIVE-NEXT: s_mov_b32 s0, 0
-; GFX1132_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX1132_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132_ITERATIVE-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132_ITERATIVE-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1132_ITERATIVE-NEXT: s_ctz_i32_b32 s2, s1
; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s2, 1, s2
+; GFX1132_ITERATIVE-NEXT: s_bitset0_b32 s1, s2
; GFX1132_ITERATIVE-NEXT: s_add_i32 s0, s0, s3
-; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s1, s1, s2
+; GFX1132_ITERATIVE-NEXT: s_cmp_lg_u32 s1, 0
; GFX1132_ITERATIVE-NEXT: s_cbranch_scc1 .LBB11_1
-; GFX1132_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132_ITERATIVE-NEXT: ; %bb.2:
; GFX1132_ITERATIVE-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132_ITERATIVE-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132_ITERATIVE-NEXT: s_xor_b32 s1, exec_lo, s1
+; GFX1132_ITERATIVE-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132_ITERATIVE-NEXT: s_cbranch_execz .LBB11_4
; GFX1132_ITERATIVE-NEXT: ; %bb.3:
-; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
-; GFX1132_ITERATIVE-NEXT: ds_sub_u32 v0, v1
+; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, 0
+; GFX1132_ITERATIVE-NEXT: ds_sub_u32 v1, v0
; GFX1132_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132_ITERATIVE-NEXT: buffer_gl0_inv
; GFX1132_ITERATIVE-NEXT: .LBB11_4:
@@ -5604,31 +5719,31 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
;
; GFX8_DPP-LABEL: sub_i32_varying_nouse:
; GFX8_DPP: ; %bb.0: ; %entry
-; GFX8_DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX8_DPP-NEXT: v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX8_DPP-NEXT: v_mbcnt_hi_u32_b32 v3, exec_hi, v3
+; GFX8_DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX8_DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
; GFX8_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX8_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]
; GFX8_DPP-NEXT: s_nop 1
-; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX8_DPP-NEXT: s_nop 1
-; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX8_DPP-NEXT: s_nop 1
-; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX8_DPP-NEXT: s_nop 1
-; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX8_DPP-NEXT: s_nop 1
-; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX8_DPP-NEXT: s_nop 1
-; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX8_DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX8_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX8_DPP-NEXT: s_mov_b32 s0, s2
-; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX8_DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX8_DPP-NEXT: s_cbranch_execz .LBB11_2
; GFX8_DPP-NEXT: ; %bb.1:
; GFX8_DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX8_DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX8_DPP-NEXT: s_mov_b32 m0, -1
; GFX8_DPP-NEXT: ds_sub_u32 v2, v0
; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -5637,31 +5752,31 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
;
; GFX9_DPP-LABEL: sub_i32_varying_nouse:
; GFX9_DPP: ; %bb.0: ; %entry
-; GFX9_DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX9_DPP-NEXT: v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9_DPP-NEXT: v_mbcnt_hi_u32_b32 v3, exec_hi, v3
+; GFX9_DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9_DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
; GFX9_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]
; GFX9_DPP-NEXT: s_nop 1
-; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9_DPP-NEXT: s_nop 1
-; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9_DPP-NEXT: s_nop 1
-; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9_DPP-NEXT: s_nop 1
-; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9_DPP-NEXT: s_nop 1
-; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9_DPP-NEXT: s_nop 1
-; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9_DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX9_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9_DPP-NEXT: s_mov_b32 s0, s2
-; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX9_DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9_DPP-NEXT: s_cbranch_execz .LBB11_2
; GFX9_DPP-NEXT: ; %bb.1:
; GFX9_DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX9_DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9_DPP-NEXT: ds_sub_u32 v2, v0
; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9_DPP-NEXT: .LBB11_2:
@@ -5671,27 +5786,34 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
; GFX1064_DPP: ; %bb.0: ; %entry
; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]
-; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1064_DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1064_DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064_DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064_DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064_DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1064_DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
; GFX1064_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064_DPP-NEXT: v_readlane_b32 s2, v1, 0
-; GFX1064_DPP-NEXT: v_readlane_b32 s3, v1, 32
+; GFX1064_DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX1064_DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v3, exec_hi, v0
-; GFX1064_DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX1064_DPP-NEXT: s_add_i32 s0, s2, s3
-; GFX1064_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064_DPP-NEXT: s_mov_b32 s0, s2
+; GFX1064_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064_DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064_DPP-NEXT: s_cbranch_execz .LBB11_2
; GFX1064_DPP-NEXT: ; %bb.1:
-; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, s0
-; GFX1064_DPP-NEXT: ds_sub_u32 v0, v3
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1064_DPP-NEXT: ds_sub_u32 v4, v0
; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064_DPP-NEXT: buffer_gl0_inv
; GFX1064_DPP-NEXT: .LBB11_2:
@@ -5701,21 +5823,24 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
; GFX1032_DPP: ; %bb.0: ; %entry
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
-; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1032_DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032_DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1032_DPP-NEXT: v_readlane_b32 s1, v1, 31
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, v1
-; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v4
-; GFX1032_DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032_DPP-NEXT: s_mov_b32 s0, s1
+; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032_DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032_DPP-NEXT: s_cbranch_execz .LBB11_2
; GFX1032_DPP-NEXT: ; %bb.1:
-; GFX1032_DPP-NEXT: ds_sub_u32 v0, v3
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032_DPP-NEXT: ds_sub_u32 v3, v0
; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032_DPP-NEXT: buffer_gl0_inv
; GFX1032_DPP-NEXT: .LBB11_2:
@@ -5727,33 +5852,41 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164_DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1164_DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164_DPP-NEXT: v_permlane64_b32 v2, v1
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164_DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, v1
-; GFX1164_DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1164_DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164_DPP-NEXT: s_mov_b32 s0, s2
+; GFX1164_DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164_DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164_DPP-NEXT: s_cbranch_execz .LBB11_2
; GFX1164_DPP-NEXT: ; %bb.1:
-; GFX1164_DPP-NEXT: ds_sub_u32 v0, v3
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164_DPP-NEXT: ds_sub_u32 v4, v0
; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164_DPP-NEXT: buffer_gl0_inv
; GFX1164_DPP-NEXT: .LBB11_2:
@@ -5766,24 +5899,26 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132_DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132_DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: v_readlane_b32 s1, v1, 31
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132_DPP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, v1
-; GFX1132_DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132_DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132_DPP-NEXT: s_mov_b32 s0, s1
+; GFX1132_DPP-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132_DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132_DPP-NEXT: s_cbranch_execz .LBB11_2
; GFX1132_DPP-NEXT: ; %bb.1:
-; GFX1132_DPP-NEXT: ds_sub_u32 v0, v3
+; GFX1132_DPP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, 0
+; GFX1132_DPP-NEXT: ds_sub_u32 v3, v0
; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132_DPP-NEXT: buffer_gl0_inv
; GFX1132_DPP-NEXT: .LBB11_2:
@@ -5795,33 +5930,40 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
; GFX1364-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1364-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1364-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1364-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1364-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1364-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1364-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1364-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1364-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1364-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1364-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1364-NEXT: s_wait_dscnt 0x0
+; GFX1364-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1364-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1364-NEXT: s_wait_dscnt 0x0
; GFX1364-NEXT: v_add_nc_u32_e32 v1, v1, v2
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1364-NEXT: v_permlane64_b32 v2, v1
; GFX1364-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1364-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1364-NEXT: v_readlane_b32 s2, v1, 63
; GFX1364-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1364-NEXT: v_mov_b32_e32 v0, 0
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1364-NEXT: v_mov_b32_e32 v3, v1
-; GFX1364-NEXT: s_mov_b64 s[0:1], exec
-; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1364-NEXT: s_mov_b32 s0, s2
+; GFX1364-NEXT: s_mov_b64 s[2:3], exec
+; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1364-NEXT: s_cbranch_execz .LBB11_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: ds_sub_u32 v0, v3
+; GFX1364-NEXT: v_mov_b32_e32 v0, s0
+; GFX1364-NEXT: v_mov_b32_e32 v4, 0
+; GFX1364-NEXT: ds_sub_u32 v4, v0
; GFX1364-NEXT: s_wait_dscnt 0x0
; GFX1364-NEXT: global_inv scope:SCOPE_SE
; GFX1364-NEXT: .LBB11_2:
@@ -5834,24 +5976,26 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1332-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1332-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1332-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1332-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1332-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1332-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1332-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1332-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1332-NEXT: s_wait_dscnt 0x0
; GFX1332-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1332-NEXT: v_readlane_b32 s1, v1, 31
; GFX1332-NEXT: s_mov_b32 exec_lo, s0
-; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1332-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, v1
-; GFX1332-NEXT: s_mov_b32 s0, exec_lo
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1332-NEXT: s_mov_b32 s0, s1
+; GFX1332-NEXT: s_mov_b32 s1, exec_lo
+; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1332-NEXT: s_cbranch_execz .LBB11_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: ds_sub_u32 v0, v3
+; GFX1332-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, 0
+; GFX1332-NEXT: ds_sub_u32 v3, v0
; GFX1332-NEXT: s_wait_dscnt 0x0
; GFX1332-NEXT: global_inv scope:SCOPE_SE
; GFX1332-NEXT: .LBB11_2:
@@ -5865,20 +6009,25 @@ entry:
define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out) {
; GFX7LESS-LABEL: sub_i64_constant:
; GFX7LESS: ; %bb.0: ; %entry
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, s3, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX7LESS-NEXT: v_mul_hi_u32 v0, 5, s2
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v1
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: v_readfirstlane_b32 s3, v0
; GFX7LESS-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB12_2
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-NEXT: s_mul_i32 s2, s2, 5
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0
+; GFX7LESS-NEXT: s_mul_i32 s6, s2, 5
+; GFX7LESS-NEXT: s_mul_i32 s2, s2, 0
+; GFX7LESS-NEXT: s_add_u32 s7, s3, s2
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, 0
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s7
; GFX7LESS-NEXT: s_mov_b32 m0, -1
-; GFX7LESS-NEXT: ds_sub_rtn_u64 v[0:1], v1, v[0:1]
+; GFX7LESS-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: .LBB12_2:
; GFX7LESS-NEXT: s_or_b64 exec, exec, s[0:1]
@@ -5898,20 +6047,25 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out) {
;
; GFX8-LABEL: sub_i64_constant:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, s3, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX8-NEXT: v_mul_hi_u32 v0, 5, s2
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX8-NEXT: v_readfirstlane_b32 s3, v0
; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB12_2
; GFX8-NEXT: ; %bb.1:
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX8-NEXT: s_mul_i32 s2, s2, 5
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_mov_b32_e32 v1, 0
+; GFX8-NEXT: s_mul_i32 s6, s2, 5
+; GFX8-NEXT: s_mul_i32 s2, s2, 0
+; GFX8-NEXT: s_add_u32 s7, s3, s2
+; GFX8-NEXT: v_mov_b32_e32 v3, 0
+; GFX8-NEXT: v_mov_b32_e32 v0, s6
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_mov_b32 m0, -1
-; GFX8-NEXT: ds_sub_rtn_u64 v[0:1], v1, v[0:1]
+; GFX8-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: .LBB12_2:
; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
@@ -5931,19 +6085,23 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out) {
;
; GFX9-LABEL: sub_i64_constant:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, s3, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB12_2
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX9-NEXT: s_mul_i32 s2, s2, 5
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: ds_sub_rtn_u64 v[0:1], v1, v[0:1]
+; GFX9-NEXT: s_mul_i32 s6, s2, 5
+; GFX9-NEXT: s_mul_hi_u32 s3, 5, s2
+; GFX9-NEXT: s_mul_i32 s2, s2, 0
+; GFX9-NEXT: s_add_u32 s7, s3, s2
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: .LBB12_2:
; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
@@ -5963,19 +6121,23 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out) {
;
; GFX1064-LABEL: sub_i64_constant:
; GFX1064: ; %bb.0: ; %entry
-; GFX1064-NEXT: s_mov_b64 s[2:3], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, s3, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1064-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB12_2
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-NEXT: v_mov_b32_e32 v1, 0
+; GFX1064-NEXT: s_mul_hi_u32 s3, 5, s2
+; GFX1064-NEXT: s_mul_i32 s6, s2, 0
; GFX1064-NEXT: s_mul_i32 s2, s2, 5
+; GFX1064-NEXT: s_add_u32 s3, s3, s6
+; GFX1064-NEXT: v_mov_b32_e32 v3, 0
; GFX1064-NEXT: v_mov_b32_e32 v0, s2
-; GFX1064-NEXT: ds_sub_rtn_u64 v[0:1], v1, v[0:1]
+; GFX1064-NEXT: v_mov_b32_e32 v1, s3
+; GFX1064-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: buffer_gl0_inv
; GFX1064-NEXT: .LBB12_2:
@@ -5996,18 +6158,22 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out) {
;
; GFX1032-LABEL: sub_i64_constant:
; GFX1032: ; %bb.0: ; %entry
-; GFX1032-NEXT: s_mov_b32 s1, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, exec_lo
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, s1, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s1, s0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB12_2
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0
-; GFX1032-NEXT: s_mul_i32 s1, s1, 5
-; GFX1032-NEXT: v_mov_b32_e32 v0, s1
-; GFX1032-NEXT: ds_sub_rtn_u64 v[0:1], v1, v[0:1]
+; GFX1032-NEXT: s_mul_hi_u32 s3, 5, s1
+; GFX1032-NEXT: s_mul_i32 s6, s1, 0
+; GFX1032-NEXT: s_mul_i32 s2, s1, 5
+; GFX1032-NEXT: s_add_u32 s3, s3, s6
+; GFX1032-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-NEXT: v_mov_b32_e32 v0, s2
+; GFX1032-NEXT: v_mov_b32_e32 v1, s3
+; GFX1032-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: buffer_gl0_inv
; GFX1032-NEXT: .LBB12_2:
@@ -6028,21 +6194,25 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out) {
;
; GFX1164-LABEL: sub_i64_constant:
; GFX1164: ; %bb.0: ; %entry
-; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, s3, v0
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1164-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB12_2
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0
+; GFX1164-NEXT: s_mul_hi_u32 s3, 5, s2
+; GFX1164-NEXT: s_mul_i32 s6, s2, 0
; GFX1164-NEXT: s_mul_i32 s2, s2, 5
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT: s_add_u32 s3, s3, s6
+; GFX1164-NEXT: v_mov_b32_e32 v3, 0
; GFX1164-NEXT: v_mov_b32_e32 v0, s2
-; GFX1164-NEXT: ds_sub_rtn_u64 v[0:1], v1, v[0:1]
+; GFX1164-NEXT: v_mov_b32_e32 v1, s3
+; GFX1164-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: buffer_gl0_inv
; GFX1164-NEXT: .LBB12_2:
@@ -6063,20 +6233,22 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out) {
;
; GFX1132-LABEL: sub_i64_constant:
; GFX1132: ; %bb.0: ; %entry
-; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, s1, 0
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX1132-NEXT: s_mov_b32 s0, exec_lo
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB12_2
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX1132-NEXT: v_mov_b32_e32 v1, 0
-; GFX1132-NEXT: s_mul_i32 s1, s1, 5
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_mov_b32_e32 v0, s1
-; GFX1132-NEXT: ds_sub_rtn_u64 v[0:1], v1, v[0:1]
+; GFX1132-NEXT: s_mul_hi_u32 s3, 5, s1
+; GFX1132-NEXT: s_mul_i32 s6, s1, 0
+; GFX1132-NEXT: s_mul_i32 s2, s1, 5
+; GFX1132-NEXT: s_add_u32 s3, s3, s6
+; GFX1132-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s2
+; GFX1132-NEXT: v_mov_b32_e32 v1, s3
+; GFX1132-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: buffer_gl0_inv
; GFX1132-NEXT: .LBB12_2:
@@ -6097,21 +6269,25 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out) {
;
; GFX1364-LABEL: sub_i64_constant:
; GFX1364: ; %bb.0: ; %entry
-; GFX1364-NEXT: s_mov_b64 s[2:3], exec
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_mov_b64 s[0:1], exec
-; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v2, s3, v0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX1364-NEXT: s_mov_b64 s[0:1], exec
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1364-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1364-NEXT: s_cbranch_execz .LBB12_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1364-NEXT: v_mov_b32_e32 v1, 0
+; GFX1364-NEXT: s_mul_hi_u32 s3, 5, s2
+; GFX1364-NEXT: s_mul_i32 s6, s2, 0
; GFX1364-NEXT: s_mul_i32 s2, s2, 5
-; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1364-NEXT: s_add_co_u32 s3, s3, s6
+; GFX1364-NEXT: v_mov_b32_e32 v3, 0
; GFX1364-NEXT: v_mov_b32_e32 v0, s2
-; GFX1364-NEXT: ds_sub_rtn_u64 v[0:1], v1, v[0:1]
+; GFX1364-NEXT: v_mov_b32_e32 v1, s3
+; GFX1364-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
; GFX1364-NEXT: s_wait_dscnt 0x0
; GFX1364-NEXT: global_inv scope:SCOPE_SE
; GFX1364-NEXT: .LBB12_2:
@@ -6132,20 +6308,22 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out) {
;
; GFX1332-LABEL: sub_i64_constant:
; GFX1332: ; %bb.0: ; %entry
-; GFX1332-NEXT: s_mov_b32 s1, exec_lo
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1332-NEXT: s_mov_b32 s0, exec_lo
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v2, s1, 0
; GFX1332-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX1332-NEXT: s_mov_b32 s0, exec_lo
; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1332-NEXT: s_cbranch_execz .LBB12_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX1332-NEXT: v_mov_b32_e32 v1, 0
-; GFX1332-NEXT: s_mul_i32 s1, s1, 5
-; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1332-NEXT: v_mov_b32_e32 v0, s1
-; GFX1332-NEXT: ds_sub_rtn_u64 v[0:1], v1, v[0:1]
+; GFX1332-NEXT: s_mul_hi_u32 s3, 5, s1
+; GFX1332-NEXT: s_mul_i32 s6, s1, 0
+; GFX1332-NEXT: s_mul_i32 s2, s1, 5
+; GFX1332-NEXT: s_add_co_u32 s3, s3, s6
+; GFX1332-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s2
+; GFX1332-NEXT: v_mov_b32_e32 v1, s3
+; GFX1332-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
; GFX1332-NEXT: s_wait_dscnt 0x0
; GFX1332-NEXT: global_inv scope:SCOPE_SE
; GFX1332-NEXT: .LBB12_2:
@@ -6173,29 +6351,30 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, i64 %subitive)
; GFX7LESS-LABEL: sub_i64_uniform:
; GFX7LESS: ; %bb.0: ; %entry
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, s7, v0
+; GFX7LESS-NEXT: s_mov_b64 s[4:5], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mul_hi_u32 v0, s2, v0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v1
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: v_readfirstlane_b32 s7, v0
; GFX7LESS-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB13_2
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mul_hi_u32 v0, s2, v0
-; GFX7LESS-NEXT: s_mul_i32 s7, s3, s6
-; GFX7LESS-NEXT: s_mul_i32 s6, s2, s6
+; GFX7LESS-NEXT: s_mul_i32 s8, s2, s6
+; GFX7LESS-NEXT: s_mul_i32 s6, s3, s6
+; GFX7LESS-NEXT: s_add_u32 s9, s7, s6
; GFX7LESS-NEXT: v_mov_b32_e32 v3, 0
-; GFX7LESS-NEXT: v_add_i32_e32 v1, vcc, s7, v0
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s8
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s9
; GFX7LESS-NEXT: s_mov_b32 m0, -1
; GFX7LESS-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: .LBB13_2:
; GFX7LESS-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_mov_b32 s4, s0
; GFX7LESS-NEXT: v_readfirstlane_b32 s0, v1
; GFX7LESS-NEXT: v_mul_lo_u32 v1, s3, v2
@@ -6215,27 +6394,30 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, i64 %subitive)
; GFX8-LABEL: sub_i64_uniform:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b64 s[6:7], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX8-NEXT: s_mov_b64 s[4:5], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, s6
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mul_hi_u32 v0, s2, v0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX8-NEXT: v_readfirstlane_b32 s7, v0
; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX8-NEXT: s_cbranch_execz .LBB13_2
; GFX8-NEXT: ; %bb.1:
-; GFX8-NEXT: s_bcnt1_i32_b64 s8, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v0, s8
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[6:7], s2, v0, 0
-; GFX8-NEXT: s_mul_i32 s6, s3, s8
+; GFX8-NEXT: s_mul_i32 s8, s2, s6
+; GFX8-NEXT: s_mul_i32 s6, s3, s6
+; GFX8-NEXT: s_add_u32 s9, s7, s6
; GFX8-NEXT: v_mov_b32_e32 v3, 0
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
+; GFX8-NEXT: v_mov_b32_e32 v1, s9
; GFX8-NEXT: s_mov_b32 m0, -1
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, s6, v1
; GFX8-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: .LBB13_2:
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_mov_b32 s4, s0
; GFX8-NEXT: s_mov_b32 s5, s1
; GFX8-NEXT: v_mul_lo_u32 v4, s3, v2
@@ -6254,23 +6436,23 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, i64 %subitive)
; GFX9-LABEL: sub_i64_uniform:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b64 s[6:7], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_cbranch_execz .LBB13_2
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_i32 s7, s3, s6
-; GFX9-NEXT: s_mul_hi_u32 s8, s2, s6
-; GFX9-NEXT: s_add_i32 s8, s8, s7
-; GFX9-NEXT: s_mul_i32 s6, s2, s6
-; GFX9-NEXT: v_mov_b32_e32 v0, s6
-; GFX9-NEXT: v_mov_b32_e32 v1, s8
+; GFX9-NEXT: s_mul_i32 s8, s2, s6
+; GFX9-NEXT: s_mul_hi_u32 s7, s2, s6
+; GFX9-NEXT: s_mul_i32 s6, s3, s6
+; GFX9-NEXT: s_add_u32 s9, s7, s6
; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v0, s8
+; GFX9-NEXT: v_mov_b32_e32 v1, s9
; GFX9-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: .LBB13_2:
@@ -6293,23 +6475,23 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, i64 %subitive)
; GFX1064-LABEL: sub_i64_uniform:
; GFX1064: ; %bb.0: ; %entry
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1064-NEXT: s_mov_b64 s[6:7], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], exec
+; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1064-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB13_2
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_mul_i32 s7, s3, s6
-; GFX1064-NEXT: s_mul_hi_u32 s8, s2, s6
+; GFX1064-NEXT: s_mul_hi_u32 s7, s2, s6
+; GFX1064-NEXT: s_mul_i32 s8, s3, s6
; GFX1064-NEXT: s_mul_i32 s6, s2, s6
-; GFX1064-NEXT: s_add_i32 s8, s8, s7
+; GFX1064-NEXT: s_add_u32 s7, s7, s8
+; GFX1064-NEXT: v_mov_b32_e32 v3, 0
; GFX1064-NEXT: v_mov_b32_e32 v0, s6
-; GFX1064-NEXT: v_mov_b32_e32 v1, s8
+; GFX1064-NEXT: v_mov_b32_e32 v1, s7
; GFX1064-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: buffer_gl0_inv
@@ -6331,21 +6513,21 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, i64 %subitive)
; GFX1032-LABEL: sub_i64_uniform:
; GFX1032: ; %bb.0: ; %entry
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1032-NEXT: s_mov_b32 s6, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s4, exec_lo
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s4
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB13_2
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s6
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_mul_i32 s6, s3, s5
; GFX1032-NEXT: s_mul_hi_u32 s7, s2, s5
-; GFX1032-NEXT: s_mul_i32 s5, s2, s5
-; GFX1032-NEXT: s_add_i32 s7, s7, s6
-; GFX1032-NEXT: v_mov_b32_e32 v0, s5
+; GFX1032-NEXT: s_mul_i32 s8, s3, s5
+; GFX1032-NEXT: s_mul_i32 s6, s2, s5
+; GFX1032-NEXT: s_add_u32 s7, s7, s8
+; GFX1032-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-NEXT: v_mov_b32_e32 v0, s6
; GFX1032-NEXT: v_mov_b32_e32 v1, s7
; GFX1032-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
@@ -6368,24 +6550,25 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, i64 %subitive)
; GFX1164-LABEL: sub_i64_uniform:
; GFX1164: ; %bb.0: ; %entry
; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1164-NEXT: s_mov_b64 s[6:7], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[4:5], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1164-NEXT: s_mov_b64 s[4:5], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1164-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB13_2
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1164-NEXT: v_mov_b32_e32 v3, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_mul_i32 s7, s3, s6
-; GFX1164-NEXT: s_mul_hi_u32 s8, s2, s6
+; GFX1164-NEXT: s_mul_hi_u32 s7, s2, s6
+; GFX1164-NEXT: s_mul_i32 s8, s3, s6
; GFX1164-NEXT: s_mul_i32 s6, s2, s6
-; GFX1164-NEXT: s_add_i32 s8, s8, s7
+; GFX1164-NEXT: s_add_u32 s7, s7, s8
+; GFX1164-NEXT: v_mov_b32_e32 v3, 0
; GFX1164-NEXT: v_mov_b32_e32 v0, s6
-; GFX1164-NEXT: v_mov_b32_e32 v1, s8
+; GFX1164-NEXT: v_mov_b32_e32 v1, s7
; GFX1164-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: buffer_gl0_inv
@@ -6407,23 +6590,22 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, i64 %subitive)
; GFX1132-LABEL: sub_i64_uniform:
; GFX1132: ; %bb.0: ; %entry
; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1132-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s4, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_bcnt1_i32_b32 s5, s4
+; GFX1132-NEXT: s_mov_b32 s4, exec_lo
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB13_2
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s5, s6
-; GFX1132-NEXT: v_mov_b32_e32 v3, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_mul_i32 s6, s3, s5
; GFX1132-NEXT: s_mul_hi_u32 s7, s2, s5
-; GFX1132-NEXT: s_mul_i32 s5, s2, s5
-; GFX1132-NEXT: s_add_i32 s7, s7, s6
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v1, s7
+; GFX1132-NEXT: s_mul_i32 s8, s3, s5
+; GFX1132-NEXT: s_mul_i32 s6, s2, s5
+; GFX1132-NEXT: s_add_u32 s7, s7, s8
+; GFX1132-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s6
+; GFX1132-NEXT: v_mov_b32_e32 v1, s7
; GFX1132-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: buffer_gl0_inv
@@ -6445,21 +6627,23 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, i64 %subitive)
; GFX1364-LABEL: sub_i64_uniform:
; GFX1364: ; %bb.0: ; %entry
; GFX1364-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1364-NEXT: s_mov_b64 s[6:7], exec
-; GFX1364-NEXT: s_mov_b32 s9, 0
-; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_mov_b64 s[4:5], exec
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1364-NEXT: s_mov_b64 s[4:5], exec
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1364-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1364-NEXT: s_cbranch_execz .LBB13_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: s_bcnt1_i32_b64 s8, s[6:7]
-; GFX1364-NEXT: v_mov_b32_e32 v3, 0
; GFX1364-NEXT: s_wait_kmcnt 0x0
-; GFX1364-NEXT: s_mul_u64 s[6:7], s[2:3], s[8:9]
-; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1364-NEXT: s_mul_hi_u32 s7, s2, s6
+; GFX1364-NEXT: s_mul_i32 s8, s3, s6
+; GFX1364-NEXT: s_mul_i32 s6, s2, s6
+; GFX1364-NEXT: s_add_co_u32 s7, s7, s8
+; GFX1364-NEXT: v_mov_b32_e32 v3, 0
; GFX1364-NEXT: v_mov_b32_e32 v0, s6
; GFX1364-NEXT: v_mov_b32_e32 v1, s7
; GFX1364-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
@@ -6483,26 +6667,27 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, i64 %subitive)
; GFX1332-LABEL: sub_i64_uniform:
; GFX1332: ; %bb.0: ; %entry
; GFX1332-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1332-NEXT: s_mov_b32 s6, exec_lo
-; GFX1332-NEXT: s_mov_b32 s5, 0
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
-; GFX1332-NEXT: s_mov_b32 s7, exec_lo
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1332-NEXT: s_mov_b32 s4, exec_lo
; GFX1332-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: s_bcnt1_i32_b32 s5, s4
+; GFX1332-NEXT: s_mov_b32 s4, exec_lo
; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1332-NEXT: s_cbranch_execz .LBB13_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: s_bcnt1_i32_b32 s4, s6
; GFX1332-NEXT: s_wait_kmcnt 0x0
-; GFX1332-NEXT: s_mul_u64 s[4:5], s[2:3], s[4:5]
-; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1332-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s4
-; GFX1332-NEXT: v_mov_b32_e32 v1, s5
+; GFX1332-NEXT: s_mul_hi_u32 s7, s2, s5
+; GFX1332-NEXT: s_mul_i32 s8, s3, s5
+; GFX1332-NEXT: s_mul_i32 s6, s2, s5
+; GFX1332-NEXT: s_add_co_u32 s7, s7, s8
+; GFX1332-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s6
+; GFX1332-NEXT: v_mov_b32_e32 v1, s7
; GFX1332-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
; GFX1332-NEXT: s_wait_dscnt 0x0
; GFX1332-NEXT: global_inv scope:SCOPE_SE
; GFX1332-NEXT: .LBB13_2:
-; GFX1332-NEXT: s_or_b32 exec_lo, exec_lo, s7
+; GFX1332-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX1332-NEXT: s_wait_kmcnt 0x0
; GFX1332-NEXT: v_mad_co_u64_u32 v[3:4], null, s2, v2, 0
; GFX1332-NEXT: v_readfirstlane_b32 s2, v0
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
index 4c87e6936e314..d3d7e8131a0c1 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
@@ -23,15 +23,15 @@ define amdgpu_ps void @add_i32_constant(ptr addrspace(8) inreg %out, ptr addrspa
; GFX7-NEXT: s_and_saveexec_b64 s[8:9], s[10:11]
; GFX7-NEXT: s_cbranch_execz .LBB0_4
; GFX7-NEXT: ; %bb.1:
-; GFX7-NEXT: s_mov_b64 s[12:13], exec
-; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s12, 0
-; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s13, v0
+; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX7-NEXT: s_mov_b64 s[10:11], exec
+; GFX7-NEXT: s_bcnt1_i32_b64 s12, s[10:11]
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX7-NEXT: ; implicit-def: $vgpr1
; GFX7-NEXT: s_and_saveexec_b64 s[10:11], vcc
; GFX7-NEXT: s_cbranch_execz .LBB0_3
; GFX7-NEXT: ; %bb.2:
-; GFX7-NEXT: s_bcnt1_i32_b64 s12, s[12:13]
; GFX7-NEXT: s_mul_i32 s12, s12, 5
; GFX7-NEXT: v_mov_b32_e32 v1, s12
; GFX7-NEXT: buffer_atomic_add v1, off, s[4:7], 0 glc
@@ -60,15 +60,15 @@ define amdgpu_ps void @add_i32_constant(ptr addrspace(8) inreg %out, ptr addrspa
; GFX89-NEXT: s_and_saveexec_b64 s[8:9], s[10:11]
; GFX89-NEXT: s_cbranch_execz .LBB0_4
; GFX89-NEXT: ; %bb.1:
-; GFX89-NEXT: s_mov_b64 s[12:13], exec
-; GFX89-NEXT: v_mbcnt_lo_u32_b32 v0, s12, 0
-; GFX89-NEXT: v_mbcnt_hi_u32_b32 v0, s13, v0
+; GFX89-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX89-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX89-NEXT: s_mov_b64 s[10:11], exec
+; GFX89-NEXT: s_bcnt1_i32_b64 s12, s[10:11]
; GFX89-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX89-NEXT: ; implicit-def: $vgpr1
; GFX89-NEXT: s_and_saveexec_b64 s[10:11], vcc
; GFX89-NEXT: s_cbranch_execz .LBB0_3
; GFX89-NEXT: ; %bb.2:
-; GFX89-NEXT: s_bcnt1_i32_b64 s12, s[12:13]
; GFX89-NEXT: s_mul_i32 s12, s12, 5
; GFX89-NEXT: v_mov_b32_e32 v1, s12
; GFX89-NEXT: buffer_atomic_add v1, off, s[4:7], 0 glc
@@ -97,15 +97,15 @@ define amdgpu_ps void @add_i32_constant(ptr addrspace(8) inreg %out, ptr addrspa
; GFX1064-NEXT: s_and_saveexec_b64 s[8:9], s[10:11]
; GFX1064-NEXT: s_cbranch_execz .LBB0_4
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_mov_b64 s[12:13], exec
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[10:11], exec
; GFX1064-NEXT: ; implicit-def: $vgpr1
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s12, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s13, v0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s12, s[10:11]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-NEXT: s_and_saveexec_b64 s[10:11], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB0_3
; GFX1064-NEXT: ; %bb.2:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s12, s[12:13]
; GFX1064-NEXT: s_mul_i32 s12, s12, 5
; GFX1064-NEXT: v_mov_b32_e32 v1, s12
; GFX1064-NEXT: buffer_atomic_add v1, off, s[4:7], 0 glc
@@ -135,14 +135,14 @@ define amdgpu_ps void @add_i32_constant(ptr addrspace(8) inreg %out, ptr addrspa
; GFX1032-NEXT: s_and_saveexec_b32 s8, s9
; GFX1032-NEXT: s_cbranch_execz .LBB0_4
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_mov_b32 s10, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s9, exec_lo
; GFX1032-NEXT: ; implicit-def: $vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s10, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s10, s9
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s9, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB0_3
; GFX1032-NEXT: ; %bb.2:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s10, s10
; GFX1032-NEXT: s_mul_i32 s10, s10, 5
; GFX1032-NEXT: v_mov_b32_e32 v1, s10
; GFX1032-NEXT: buffer_atomic_add v1, off, s[4:7], 0 glc
@@ -173,18 +173,19 @@ define amdgpu_ps void @add_i32_constant(ptr addrspace(8) inreg %out, ptr addrspa
; GFX1164-NEXT: s_and_saveexec_b64 s[8:9], s[10:11]
; GFX1164-NEXT: s_cbranch_execz .LBB0_4
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_mov_b64 s[12:13], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[10:11], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s12, 0
; GFX1164-NEXT: ; implicit-def: $vgpr1
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s13, v0
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_bcnt1_i32_b64 s12, s[10:11]
+; GFX1164-NEXT: s_mov_b64 s[10:11], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-NEXT: s_cbranch_execz .LBB0_3
; GFX1164-NEXT: ; %bb.2:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s12, s[12:13]
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: s_mul_i32 s12, s12, 5
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164-NEXT: v_mov_b32_e32 v1, s12
; GFX1164-NEXT: buffer_atomic_add_u32 v1, off, s[4:7], 0 glc
; GFX1164-NEXT: .LBB0_3:
@@ -216,17 +217,17 @@ define amdgpu_ps void @add_i32_constant(ptr addrspace(8) inreg %out, ptr addrspa
; GFX1132-NEXT: s_and_saveexec_b32 s8, s9
; GFX1132-NEXT: s_cbranch_execz .LBB0_4
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_mov_b32 s10, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s9, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s10, 0
; GFX1132-NEXT: ; implicit-def: $vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_bcnt1_i32_b32 s10, s9
+; GFX1132-NEXT: s_mov_b32 s9, exec_lo
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB0_3
; GFX1132-NEXT: ; %bb.2:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s10, s10
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1132-NEXT: s_mul_i32 s10, s10, 5
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1132-NEXT: v_mov_b32_e32 v1, s10
; GFX1132-NEXT: buffer_atomic_add_u32 v1, off, s[4:7], 0 glc
; GFX1132-NEXT: .LBB0_3:
@@ -258,18 +259,19 @@ define amdgpu_ps void @add_i32_constant(ptr addrspace(8) inreg %out, ptr addrspa
; GFX1364-NEXT: s_and_saveexec_b64 s[8:9], s[10:11]
; GFX1364-NEXT: s_cbranch_execz .LBB0_4
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: s_mov_b64 s[12:13], exec
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_mov_b64 s[10:11], exec
-; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, s12, 0
; GFX1364-NEXT: ; implicit-def: $vgpr1
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, s13, v0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_bcnt1_i32_b64 s12, s[10:11]
+; GFX1364-NEXT: s_mov_b64 s[10:11], exec
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1364-NEXT: s_cbranch_execz .LBB0_3
; GFX1364-NEXT: ; %bb.2:
-; GFX1364-NEXT: s_bcnt1_i32_b64 s12, s[12:13]
-; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1364-NEXT: s_mul_i32 s12, s12, 5
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1364-NEXT: v_mov_b32_e32 v1, s12
; GFX1364-NEXT: buffer_atomic_add_u32 v1, off, s[4:7], null th:TH_ATOMIC_RETURN
; GFX1364-NEXT: .LBB0_3:
@@ -301,17 +303,17 @@ define amdgpu_ps void @add_i32_constant(ptr addrspace(8) inreg %out, ptr addrspa
; GFX1332-NEXT: s_and_saveexec_b32 s8, s9
; GFX1332-NEXT: s_cbranch_execz .LBB0_4
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: s_mov_b32 s10, exec_lo
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1332-NEXT: s_mov_b32 s9, exec_lo
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, s10, 0
; GFX1332-NEXT: ; implicit-def: $vgpr1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: s_bcnt1_i32_b32 s10, s9
+; GFX1332-NEXT: s_mov_b32 s9, exec_lo
; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1332-NEXT: s_cbranch_execz .LBB0_3
; GFX1332-NEXT: ; %bb.2:
-; GFX1332-NEXT: s_bcnt1_i32_b32 s10, s10
-; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1332-NEXT: s_mul_i32 s10, s10, 5
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1332-NEXT: v_mov_b32_e32 v1, s10
; GFX1332-NEXT: buffer_atomic_add_u32 v1, off, s[4:7], null th:TH_ATOMIC_RETURN
; GFX1332-NEXT: .LBB0_3:
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
index debb51db298c1..cd54db8122b79 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
@@ -20,16 +20,16 @@ declare i32 @llvm.amdgcn.raw.ptr.buffer.atomic.sub(i32, ptr addrspace(8), i32, i
define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace(8) %inout) {
; GFX6-LABEL: add_i32_constant:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_mov_b64 s[2:3], exec
-; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX6-NEXT: s_mov_b64 s[0:1], exec
+; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX6-NEXT: ; implicit-def: $vgpr1
; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX6-NEXT: s_cbranch_execz .LBB0_2
; GFX6-NEXT: ; %bb.1:
; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd
-; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX6-NEXT: s_mul_i32 s2, s2, 5
; GFX6-NEXT: v_mov_b32_e32 v1, s2
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
@@ -48,16 +48,16 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX8-LABEL: add_i32_constant:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB0_2
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX8-NEXT: s_mul_i32 s2, s2, 5
; GFX8-NEXT: v_mov_b32_e32 v1, s2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
@@ -76,16 +76,16 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX9-LABEL: add_i32_constant:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB0_2
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_mul_i32 s2, s2, 5
; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -103,16 +103,16 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX10W64-LABEL: add_i32_constant:
; GFX10W64: ; %bb.0: ; %entry
-; GFX10W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W64-NEXT: s_mov_b64 s[0:1], exec
; GFX10W64-NEXT: ; implicit-def: $vgpr1
-; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX10W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX10W64-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX10W64-NEXT: s_cbranch_execz .LBB0_2
; GFX10W64-NEXT: ; %bb.1:
; GFX10W64-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX10W64-NEXT: s_mul_i32 s2, s2, 5
; GFX10W64-NEXT: v_mov_b32_e32 v1, s2
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
@@ -131,15 +131,15 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX10W32-LABEL: add_i32_constant:
; GFX10W32: ; %bb.0: ; %entry
-; GFX10W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W32-NEXT: s_mov_b32 s0, exec_lo
; GFX10W32-NEXT: ; implicit-def: $vgpr1
-; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
+; GFX10W32-NEXT: s_bcnt1_i32_b32 s1, s0
; GFX10W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10W32-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX10W32-NEXT: s_cbranch_execz .LBB0_2
; GFX10W32-NEXT: ; %bb.1:
; GFX10W32-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W32-NEXT: s_bcnt1_i32_b32 s1, s1
; GFX10W32-NEXT: s_mul_i32 s1, s1, 5
; GFX10W32-NEXT: v_mov_b32_e32 v1, s1
; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)
@@ -158,19 +158,20 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX11W64-LABEL: add_i32_constant:
; GFX11W64: ; %bb.0: ; %entry
-; GFX11W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W64-NEXT: ; implicit-def: $vgpr1
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W64-NEXT: s_cbranch_execz .LBB0_2
; GFX11W64-NEXT: ; %bb.1:
; GFX11W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11W64-NEXT: s_mul_i32 s2, s2, 5
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11W64-NEXT: v_mov_b32_e32 v1, s2
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W64-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], 0 glc
@@ -188,18 +189,18 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX11W32-LABEL: add_i32_constant:
; GFX11W32: ; %bb.0: ; %entry
-; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX11W32-NEXT: ; implicit-def: $vgpr1
-; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX11W32-NEXT: s_mov_b32 s0, exec_lo
; GFX11W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W32-NEXT: s_cbranch_execz .LBB0_2
; GFX11W32-NEXT: ; %bb.1:
; GFX11W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11W32-NEXT: s_mul_i32 s1, s1, 5
+; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11W32-NEXT: v_mov_b32_e32 v1, s1
; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W32-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], 0 glc
@@ -217,20 +218,20 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX12W64-LABEL: add_i32_constant:
; GFX12W64: ; %bb.0: ; %entry
-; GFX12W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W64-NEXT: ; implicit-def: $vgpr1
-; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W64-NEXT: s_cbranch_execz .LBB0_2
; GFX12W64-NEXT: ; %bb.1:
; GFX12W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W64-NEXT: s_mul_i32 s2, s2, 5
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12W64-NEXT: v_mov_b32_e32 v1, s2
; GFX12W64-NEXT: s_wait_kmcnt 0x0
; GFX12W64-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -249,19 +250,18 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX12W32-LABEL: add_i32_constant:
; GFX12W32: ; %bb.0: ; %entry
-; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX12W32-NEXT: ; implicit-def: $vgpr1
-; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX12W32-NEXT: s_mov_b32 s0, exec_lo
; GFX12W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W32-NEXT: s_cbranch_execz .LBB0_2
; GFX12W32-NEXT: ; %bb.1:
; GFX12W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W32-NEXT: s_mul_i32 s1, s1, 5
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12W32-NEXT: v_mov_b32_e32 v1, s1
; GFX12W32-NEXT: s_wait_kmcnt 0x0
; GFX12W32-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -279,19 +279,20 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX13W64-LABEL: add_i32_constant:
; GFX13W64: ; %bb.0: ; %entry
-; GFX13W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W64-NEXT: ; implicit-def: $vgpr1
-; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX13W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W64-NEXT: s_cbranch_execz .LBB0_2
; GFX13W64-NEXT: ; %bb.1:
; GFX13W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX13W64-NEXT: s_mul_i32 s2, s2, 5
+; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX13W64-NEXT: v_mov_b32_e32 v1, s2
; GFX13W64-NEXT: s_wait_kmcnt 0x0
; GFX13W64-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -309,18 +310,18 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX13W32-LABEL: add_i32_constant:
; GFX13W32: ; %bb.0: ; %entry
-; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX13W32-NEXT: ; implicit-def: $vgpr1
-; GFX13W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX13W32-NEXT: s_mov_b32 s0, exec_lo
; GFX13W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W32-NEXT: s_cbranch_execz .LBB0_2
; GFX13W32-NEXT: ; %bb.1:
; GFX13W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX13W32-NEXT: s_mul_i32 s1, s1, 5
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX13W32-NEXT: v_mov_b32_e32 v1, s1
; GFX13W32-NEXT: s_wait_kmcnt 0x0
; GFX13W32-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -344,26 +345,26 @@ entry:
define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(8) %inout, i32 %additive) {
; GFX6-LABEL: add_i32_uniform:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0x11
-; GFX6-NEXT: s_mov_b64 s[2:3], exec
-; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX6-NEXT: s_load_dword s2, s[4:5], 0x11
+; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX6-NEXT: s_mov_b64 s[0:1], exec
+; GFX6-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX6-NEXT: ; implicit-def: $vgpr1
; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX6-NEXT: s_cbranch_execz .LBB1_2
; GFX6-NEXT: ; %bb.1:
; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd
-; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mul_i32 s2, s6, s2
-; GFX6-NEXT: v_mov_b32_e32 v1, s2
+; GFX6-NEXT: s_mul_i32 s3, s2, s3
+; GFX6-NEXT: v_mov_b32_e32 v1, s3
; GFX6-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
; GFX6-NEXT: .LBB1_2:
; GFX6-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX6-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_readfirstlane_b32 s4, v1
; GFX6-NEXT: s_mov_b32 s3, 0xf000
@@ -374,26 +375,26 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX8-LABEL: add_i32_uniform:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB1_2
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mul_i32 s2, s6, s2
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
+; GFX8-NEXT: s_mul_i32 s3, s2, s3
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
; GFX8-NEXT: .LBB1_2:
; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX8-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s2, v1
; GFX8-NEXT: v_add_u32_e32 v2, vcc, s2, v0
@@ -404,26 +405,26 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX9-LABEL: add_i32_uniform:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB1_2
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_i32 s2, s6, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: s_mul_i32 s3, s2, s3
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
; GFX9-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
; GFX9-NEXT: .LBB1_2:
; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX9-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s2, v1
; GFX9-NEXT: v_mov_b32_e32 v2, 0
@@ -433,29 +434,30 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX10W64-LABEL: add_i32_uniform:
; GFX10W64: ; %bb.0: ; %entry
-; GFX10W64-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX10W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX10W64-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W64-NEXT: s_mov_b64 s[0:1], exec
; GFX10W64-NEXT: ; implicit-def: $vgpr1
-; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX10W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX10W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX10W64-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX10W64-NEXT: s_cbranch_execz .LBB1_2
; GFX10W64-NEXT: ; %bb.1:
; GFX10W64-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10W64-NEXT: s_mul_i32 s2, s6, s2
-; GFX10W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX10W64-NEXT: s_mul_i32 s3, s2, s3
+; GFX10W64-NEXT: v_mov_b32_e32 v1, s3
; GFX10W64-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
; GFX10W64-NEXT: .LBB1_2:
; GFX10W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX10W64-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX10W64-NEXT: s_waitcnt vmcnt(0)
-; GFX10W64-NEXT: v_readfirstlane_b32 s2, v1
+; GFX10W64-NEXT: s_mov_b32 null, 0
+; GFX10W64-NEXT: v_readfirstlane_b32 s4, v1
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10W64-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s6, v0, s[2:3]
+; GFX10W64-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s2, v0, s[4:5]
; GFX10W64-NEXT: v_mov_b32_e32 v1, 0
; GFX10W64-NEXT: global_store_dword v1, v0, s[0:1]
; GFX10W64-NEXT: s_endpgm
@@ -463,15 +465,15 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX10W32-LABEL: add_i32_uniform:
; GFX10W32: ; %bb.0: ; %entry
; GFX10W32-NEXT: s_load_dword s0, s[4:5], 0x44
-; GFX10W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W32-NEXT: s_mov_b32 s1, exec_lo
; GFX10W32-NEXT: ; implicit-def: $vgpr1
-; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX10W32-NEXT: s_bcnt1_i32_b32 s2, s1
; GFX10W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10W32-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX10W32-NEXT: s_cbranch_execz .LBB1_2
; GFX10W32-NEXT: ; %bb.1:
; GFX10W32-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX10W32-NEXT: s_mul_i32 s2, s0, s2
; GFX10W32-NEXT: v_mov_b32_e32 v1, s2
@@ -491,31 +493,32 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX11W64-LABEL: add_i32_uniform:
; GFX11W64: ; %bb.0: ; %entry
-; GFX11W64-NEXT: s_load_b32 s6, s[4:5], 0x44
-; GFX11W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX11W64-NEXT: s_load_b32 s2, s[4:5], 0x44
+; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W64-NEXT: ; implicit-def: $vgpr1
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W64-NEXT: s_cbranch_execz .LBB1_2
; GFX11W64-NEXT: ; %bb.1:
; GFX11W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11W64-NEXT: s_mul_i32 s2, s6, s2
+; GFX11W64-NEXT: s_mul_i32 s3, s2, s3
; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX11W64-NEXT: v_mov_b32_e32 v1, s3
; GFX11W64-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], 0 glc
; GFX11W64-NEXT: .LBB1_2:
; GFX11W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX11W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX11W64-NEXT: s_waitcnt vmcnt(0)
-; GFX11W64-NEXT: v_readfirstlane_b32 s2, v1
+; GFX11W64-NEXT: v_readfirstlane_b32 s4, v1
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11W64-NEXT: v_mad_u64_u32 v[1:2], null, s6, v0, s[2:3]
+; GFX11W64-NEXT: v_mad_u64_u32 v[1:2], null, s2, v0, s[4:5]
; GFX11W64-NEXT: v_mov_b32_e32 v0, 0
; GFX11W64-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX11W64-NEXT: s_endpgm
@@ -523,16 +526,16 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX11W32-LABEL: add_i32_uniform:
; GFX11W32: ; %bb.0: ; %entry
; GFX11W32-NEXT: s_load_b32 s0, s[4:5], 0x44
-; GFX11W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W32-NEXT: ; implicit-def: $vgpr1
-; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
; GFX11W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W32-NEXT: s_cbranch_execz .LBB1_2
; GFX11W32-NEXT: ; %bb.1:
; GFX11W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W32-NEXT: s_mul_i32 s2, s0, s2
; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -552,33 +555,32 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX12W64-LABEL: add_i32_uniform:
; GFX12W64: ; %bb.0: ; %entry
-; GFX12W64-NEXT: s_load_b32 s6, s[4:5], 0x44
-; GFX12W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX12W64-NEXT: s_load_b32 s2, s[4:5], 0x44
+; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W64-NEXT: ; implicit-def: $vgpr1
-; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W64-NEXT: s_cbranch_execz .LBB1_2
; GFX12W64-NEXT: ; %bb.1:
; GFX12W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX12W64-NEXT: s_wait_kmcnt 0x0
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12W64-NEXT: s_mul_i32 s2, s6, s2
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX12W64-NEXT: s_mul_i32 s3, s2, s3
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12W64-NEXT: v_mov_b32_e32 v1, s3
; GFX12W64-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
; GFX12W64-NEXT: .LBB1_2:
; GFX12W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX12W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX12W64-NEXT: s_wait_loadcnt 0x0
-; GFX12W64-NEXT: v_readfirstlane_b32 s2, v1
+; GFX12W64-NEXT: v_readfirstlane_b32 s4, v1
; GFX12W64-NEXT: s_wait_kmcnt 0x0
-; GFX12W64-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12W64-NEXT: v_mad_co_u64_u32 v[0:1], null, s6, v0, s[2:3]
+; GFX12W64-NEXT: v_mad_co_u64_u32 v[0:1], null, s2, v0, s[4:5]
; GFX12W64-NEXT: v_mov_b32_e32 v1, 0
; GFX12W64-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX12W64-NEXT: s_endpgm
@@ -586,20 +588,19 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX12W32-LABEL: add_i32_uniform:
; GFX12W32: ; %bb.0: ; %entry
; GFX12W32-NEXT: s_load_b32 s0, s[4:5], 0x44
-; GFX12W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W32-NEXT: ; implicit-def: $vgpr1
-; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
; GFX12W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W32-NEXT: s_cbranch_execz .LBB1_2
; GFX12W32-NEXT: ; %bb.1:
; GFX12W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX12W32-NEXT: s_wait_kmcnt 0x0
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W32-NEXT: s_mul_i32 s2, s0, s2
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12W32-NEXT: v_mov_b32_e32 v1, s2
; GFX12W32-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
; GFX12W32-NEXT: .LBB1_2:
@@ -616,31 +617,32 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX13W64-LABEL: add_i32_uniform:
; GFX13W64: ; %bb.0: ; %entry
-; GFX13W64-NEXT: s_load_b32 s6, s[4:5], 0x44 nv
-; GFX13W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX13W64-NEXT: s_load_b32 s2, s[4:5], 0x44 nv
+; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W64-NEXT: ; implicit-def: $vgpr1
-; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX13W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W64-NEXT: s_cbranch_execz .LBB1_2
; GFX13W64-NEXT: ; %bb.1:
; GFX13W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX13W64-NEXT: s_wait_kmcnt 0x0
-; GFX13W64-NEXT: s_mul_i32 s2, s6, s2
+; GFX13W64-NEXT: s_mul_i32 s3, s2, s3
; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX13W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX13W64-NEXT: v_mov_b32_e32 v1, s3
; GFX13W64-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
; GFX13W64-NEXT: .LBB1_2:
; GFX13W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX13W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX13W64-NEXT: s_wait_loadcnt 0x0
-; GFX13W64-NEXT: v_readfirstlane_b32 s2, v1
+; GFX13W64-NEXT: v_readfirstlane_b32 s4, v1
; GFX13W64-NEXT: s_wait_kmcnt 0x0
; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13W64-NEXT: v_mad_co_u64_u32 v[0:1], null, s6, v0, s[2:3]
+; GFX13W64-NEXT: v_mad_co_u64_u32 v[0:1], null, s2, v0, s[4:5]
; GFX13W64-NEXT: v_mov_b32_e32 v1, 0
; GFX13W64-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX13W64-NEXT: s_endpgm
@@ -648,16 +650,16 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX13W32-LABEL: add_i32_uniform:
; GFX13W32: ; %bb.0: ; %entry
; GFX13W32-NEXT: s_load_b32 s0, s[4:5], 0x44 nv
-; GFX13W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W32-NEXT: ; implicit-def: $vgpr1
-; GFX13W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
; GFX13W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W32-NEXT: s_cbranch_execz .LBB1_2
; GFX13W32-NEXT: ; %bb.1:
; GFX13W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX13W32-NEXT: s_wait_kmcnt 0x0
; GFX13W32-NEXT: s_mul_i32 s2, s0, s2
; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -1269,16 +1271,16 @@ entry:
define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace(8) %inout) {
; GFX6-LABEL: sub_i32_constant:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_mov_b64 s[2:3], exec
-; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX6-NEXT: s_mov_b64 s[0:1], exec
+; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX6-NEXT: ; implicit-def: $vgpr1
; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX6-NEXT: s_cbranch_execz .LBB4_2
; GFX6-NEXT: ; %bb.1:
; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd
-; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX6-NEXT: s_mul_i32 s2, s2, 5
; GFX6-NEXT: v_mov_b32_e32 v1, s2
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
@@ -1298,16 +1300,16 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX8-LABEL: sub_i32_constant:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB4_2
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX8-NEXT: s_mul_i32 s2, s2, 5
; GFX8-NEXT: v_mov_b32_e32 v1, s2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
@@ -1327,16 +1329,16 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX9-LABEL: sub_i32_constant:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB4_2
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_mul_i32 s2, s2, 5
; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -1355,16 +1357,16 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX10W64-LABEL: sub_i32_constant:
; GFX10W64: ; %bb.0: ; %entry
-; GFX10W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W64-NEXT: s_mov_b64 s[0:1], exec
; GFX10W64-NEXT: ; implicit-def: $vgpr1
-; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX10W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX10W64-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX10W64-NEXT: s_cbranch_execz .LBB4_2
; GFX10W64-NEXT: ; %bb.1:
; GFX10W64-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX10W64-NEXT: s_mul_i32 s2, s2, 5
; GFX10W64-NEXT: v_mov_b32_e32 v1, s2
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
@@ -1384,15 +1386,15 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX10W32-LABEL: sub_i32_constant:
; GFX10W32: ; %bb.0: ; %entry
-; GFX10W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W32-NEXT: s_mov_b32 s0, exec_lo
; GFX10W32-NEXT: ; implicit-def: $vgpr1
-; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
+; GFX10W32-NEXT: s_bcnt1_i32_b32 s1, s0
; GFX10W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10W32-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX10W32-NEXT: s_cbranch_execz .LBB4_2
; GFX10W32-NEXT: ; %bb.1:
; GFX10W32-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W32-NEXT: s_bcnt1_i32_b32 s1, s1
; GFX10W32-NEXT: s_mul_i32 s1, s1, 5
; GFX10W32-NEXT: v_mov_b32_e32 v1, s1
; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)
@@ -1412,19 +1414,20 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX11W64-LABEL: sub_i32_constant:
; GFX11W64: ; %bb.0: ; %entry
-; GFX11W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W64-NEXT: ; implicit-def: $vgpr1
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W64-NEXT: s_cbranch_execz .LBB4_2
; GFX11W64-NEXT: ; %bb.1:
; GFX11W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11W64-NEXT: s_mul_i32 s2, s2, 5
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11W64-NEXT: v_mov_b32_e32 v1, s2
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W64-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], 0 glc
@@ -1443,18 +1446,18 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX11W32-LABEL: sub_i32_constant:
; GFX11W32: ; %bb.0: ; %entry
-; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX11W32-NEXT: ; implicit-def: $vgpr1
-; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX11W32-NEXT: s_mov_b32 s0, exec_lo
; GFX11W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W32-NEXT: s_cbranch_execz .LBB4_2
; GFX11W32-NEXT: ; %bb.1:
; GFX11W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11W32-NEXT: s_mul_i32 s1, s1, 5
+; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11W32-NEXT: v_mov_b32_e32 v1, s1
; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W32-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], 0 glc
@@ -1473,20 +1476,20 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX12W64-LABEL: sub_i32_constant:
; GFX12W64: ; %bb.0: ; %entry
-; GFX12W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W64-NEXT: ; implicit-def: $vgpr1
-; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W64-NEXT: s_cbranch_execz .LBB4_2
; GFX12W64-NEXT: ; %bb.1:
; GFX12W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W64-NEXT: s_mul_i32 s2, s2, 5
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12W64-NEXT: v_mov_b32_e32 v1, s2
; GFX12W64-NEXT: s_wait_kmcnt 0x0
; GFX12W64-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -1506,19 +1509,18 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX12W32-LABEL: sub_i32_constant:
; GFX12W32: ; %bb.0: ; %entry
-; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX12W32-NEXT: ; implicit-def: $vgpr1
-; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX12W32-NEXT: s_mov_b32 s0, exec_lo
; GFX12W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W32-NEXT: s_cbranch_execz .LBB4_2
; GFX12W32-NEXT: ; %bb.1:
; GFX12W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W32-NEXT: s_mul_i32 s1, s1, 5
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12W32-NEXT: v_mov_b32_e32 v1, s1
; GFX12W32-NEXT: s_wait_kmcnt 0x0
; GFX12W32-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -1537,19 +1539,20 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX13W64-LABEL: sub_i32_constant:
; GFX13W64: ; %bb.0: ; %entry
-; GFX13W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W64-NEXT: ; implicit-def: $vgpr1
-; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX13W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W64-NEXT: s_cbranch_execz .LBB4_2
; GFX13W64-NEXT: ; %bb.1:
; GFX13W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX13W64-NEXT: s_mul_i32 s2, s2, 5
+; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX13W64-NEXT: v_mov_b32_e32 v1, s2
; GFX13W64-NEXT: s_wait_kmcnt 0x0
; GFX13W64-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -1568,18 +1571,18 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX13W32-LABEL: sub_i32_constant:
; GFX13W32: ; %bb.0: ; %entry
-; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX13W32-NEXT: ; implicit-def: $vgpr1
-; GFX13W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX13W32-NEXT: s_mov_b32 s0, exec_lo
; GFX13W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W32-NEXT: s_cbranch_execz .LBB4_2
; GFX13W32-NEXT: ; %bb.1:
; GFX13W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX13W32-NEXT: s_mul_i32 s1, s1, 5
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX13W32-NEXT: v_mov_b32_e32 v1, s1
; GFX13W32-NEXT: s_wait_kmcnt 0x0
; GFX13W32-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -1603,26 +1606,26 @@ entry:
define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(8) %inout, i32 %subitive) {
; GFX6-LABEL: sub_i32_uniform:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0x11
-; GFX6-NEXT: s_mov_b64 s[2:3], exec
-; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX6-NEXT: s_load_dword s2, s[4:5], 0x11
+; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX6-NEXT: s_mov_b64 s[0:1], exec
+; GFX6-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX6-NEXT: ; implicit-def: $vgpr1
; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX6-NEXT: s_cbranch_execz .LBB5_2
; GFX6-NEXT: ; %bb.1:
; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd
-; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mul_i32 s2, s6, s2
-; GFX6-NEXT: v_mov_b32_e32 v1, s2
+; GFX6-NEXT: s_mul_i32 s3, s2, s3
+; GFX6-NEXT: v_mov_b32_e32 v1, s3
; GFX6-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc
; GFX6-NEXT: .LBB5_2:
; GFX6-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX6-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_readfirstlane_b32 s4, v1
; GFX6-NEXT: s_mov_b32 s3, 0xf000
@@ -1633,26 +1636,26 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX8-LABEL: sub_i32_uniform:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB5_2
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mul_i32 s2, s6, s2
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
+; GFX8-NEXT: s_mul_i32 s3, s2, s3
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc
; GFX8-NEXT: .LBB5_2:
; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX8-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s2, v1
; GFX8-NEXT: v_sub_u32_e32 v2, vcc, s2, v0
@@ -1663,26 +1666,26 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX9-LABEL: sub_i32_uniform:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB5_2
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_i32 s2, s6, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: s_mul_i32 s3, s2, s3
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
; GFX9-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc
; GFX9-NEXT: .LBB5_2:
; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX9-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s2, v1
; GFX9-NEXT: v_mov_b32_e32 v2, 0
@@ -1692,27 +1695,27 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX10W64-LABEL: sub_i32_uniform:
; GFX10W64: ; %bb.0: ; %entry
-; GFX10W64-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX10W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX10W64-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W64-NEXT: s_mov_b64 s[0:1], exec
; GFX10W64-NEXT: ; implicit-def: $vgpr1
-; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX10W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX10W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX10W64-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX10W64-NEXT: s_cbranch_execz .LBB5_2
; GFX10W64-NEXT: ; %bb.1:
; GFX10W64-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10W64-NEXT: s_mul_i32 s2, s6, s2
-; GFX10W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX10W64-NEXT: s_mul_i32 s3, s2, s3
+; GFX10W64-NEXT: v_mov_b32_e32 v1, s3
; GFX10W64-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc
; GFX10W64-NEXT: .LBB5_2:
; GFX10W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX10W64-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10W64-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX10W64-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX10W64-NEXT: s_waitcnt vmcnt(0)
; GFX10W64-NEXT: v_readfirstlane_b32 s2, v1
; GFX10W64-NEXT: v_mov_b32_e32 v1, 0
@@ -1723,15 +1726,15 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX10W32-LABEL: sub_i32_uniform:
; GFX10W32: ; %bb.0: ; %entry
; GFX10W32-NEXT: s_load_dword s0, s[4:5], 0x44
-; GFX10W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W32-NEXT: s_mov_b32 s1, exec_lo
; GFX10W32-NEXT: ; implicit-def: $vgpr1
-; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX10W32-NEXT: s_bcnt1_i32_b32 s2, s1
; GFX10W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10W32-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX10W32-NEXT: s_cbranch_execz .LBB5_2
; GFX10W32-NEXT: ; %bb.1:
; GFX10W32-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX10W32-NEXT: s_mul_i32 s2, s0, s2
; GFX10W32-NEXT: v_mov_b32_e32 v1, s2
@@ -1751,28 +1754,29 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX11W64-LABEL: sub_i32_uniform:
; GFX11W64: ; %bb.0: ; %entry
-; GFX11W64-NEXT: s_load_b32 s6, s[4:5], 0x44
-; GFX11W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX11W64-NEXT: s_load_b32 s2, s[4:5], 0x44
+; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W64-NEXT: ; implicit-def: $vgpr1
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W64-NEXT: s_cbranch_execz .LBB5_2
; GFX11W64-NEXT: ; %bb.1:
; GFX11W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11W64-NEXT: s_mul_i32 s2, s6, s2
+; GFX11W64-NEXT: s_mul_i32 s3, s2, s3
; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX11W64-NEXT: v_mov_b32_e32 v1, s3
; GFX11W64-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], 0 glc
; GFX11W64-NEXT: .LBB5_2:
; GFX11W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX11W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11W64-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX11W64-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX11W64-NEXT: s_waitcnt vmcnt(0)
; GFX11W64-NEXT: v_readfirstlane_b32 s2, v1
; GFX11W64-NEXT: v_mov_b32_e32 v1, 0
@@ -1784,16 +1788,16 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX11W32-LABEL: sub_i32_uniform:
; GFX11W32: ; %bb.0: ; %entry
; GFX11W32-NEXT: s_load_b32 s0, s[4:5], 0x44
-; GFX11W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W32-NEXT: ; implicit-def: $vgpr1
-; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
; GFX11W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W32-NEXT: s_cbranch_execz .LBB5_2
; GFX11W32-NEXT: ; %bb.1:
; GFX11W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W32-NEXT: s_mul_i32 s2, s0, s2
; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -1814,29 +1818,29 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX12W64-LABEL: sub_i32_uniform:
; GFX12W64: ; %bb.0: ; %entry
-; GFX12W64-NEXT: s_load_b32 s6, s[4:5], 0x44
-; GFX12W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX12W64-NEXT: s_load_b32 s2, s[4:5], 0x44
+; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W64-NEXT: ; implicit-def: $vgpr1
-; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W64-NEXT: s_cbranch_execz .LBB5_2
; GFX12W64-NEXT: ; %bb.1:
; GFX12W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX12W64-NEXT: s_wait_kmcnt 0x0
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12W64-NEXT: s_mul_i32 s2, s6, s2
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX12W64-NEXT: s_mul_i32 s3, s2, s3
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12W64-NEXT: v_mov_b32_e32 v1, s3
; GFX12W64-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
; GFX12W64-NEXT: .LBB5_2:
; GFX12W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX12W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX12W64-NEXT: s_wait_kmcnt 0x0
-; GFX12W64-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX12W64-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX12W64-NEXT: s_wait_loadcnt 0x0
; GFX12W64-NEXT: v_readfirstlane_b32 s2, v1
; GFX12W64-NEXT: v_mov_b32_e32 v1, 0
@@ -1849,20 +1853,19 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX12W32-LABEL: sub_i32_uniform:
; GFX12W32: ; %bb.0: ; %entry
; GFX12W32-NEXT: s_load_b32 s0, s[4:5], 0x44
-; GFX12W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W32-NEXT: ; implicit-def: $vgpr1
-; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
; GFX12W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W32-NEXT: s_cbranch_execz .LBB5_2
; GFX12W32-NEXT: ; %bb.1:
; GFX12W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX12W32-NEXT: s_wait_kmcnt 0x0
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W32-NEXT: s_mul_i32 s2, s0, s2
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12W32-NEXT: v_mov_b32_e32 v1, s2
; GFX12W32-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
; GFX12W32-NEXT: .LBB5_2:
@@ -1881,28 +1884,29 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX13W64-LABEL: sub_i32_uniform:
; GFX13W64: ; %bb.0: ; %entry
-; GFX13W64-NEXT: s_load_b32 s6, s[4:5], 0x44 nv
-; GFX13W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX13W64-NEXT: s_load_b32 s2, s[4:5], 0x44 nv
+; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W64-NEXT: ; implicit-def: $vgpr1
-; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX13W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W64-NEXT: s_cbranch_execz .LBB5_2
; GFX13W64-NEXT: ; %bb.1:
; GFX13W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX13W64-NEXT: s_wait_kmcnt 0x0
-; GFX13W64-NEXT: s_mul_i32 s2, s6, s2
+; GFX13W64-NEXT: s_mul_i32 s3, s2, s3
; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX13W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX13W64-NEXT: v_mov_b32_e32 v1, s3
; GFX13W64-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
; GFX13W64-NEXT: .LBB5_2:
; GFX13W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX13W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX13W64-NEXT: s_wait_kmcnt 0x0
-; GFX13W64-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX13W64-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX13W64-NEXT: s_wait_loadcnt 0x0
; GFX13W64-NEXT: v_readfirstlane_b32 s2, v1
; GFX13W64-NEXT: v_mov_b32_e32 v1, 0
@@ -1914,16 +1918,16 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX13W32-LABEL: sub_i32_uniform:
; GFX13W32: ; %bb.0: ; %entry
; GFX13W32-NEXT: s_load_b32 s0, s[4:5], 0x44 nv
-; GFX13W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W32-NEXT: ; implicit-def: $vgpr1
-; GFX13W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
; GFX13W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W32-NEXT: s_cbranch_execz .LBB5_2
; GFX13W32-NEXT: ; %bb.1:
; GFX13W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX13W32-NEXT: s_wait_kmcnt 0x0
; GFX13W32-NEXT: s_mul_i32 s2, s0, s2
; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
index c140252b6003d..cb6b4cd7a09d7 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
@@ -20,16 +20,16 @@ declare i32 @llvm.amdgcn.struct.ptr.buffer.atomic.sub(i32, ptr addrspace(8), i32
define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace(8) %inout) {
; GFX6-LABEL: add_i32_constant:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_mov_b64 s[2:3], exec
-; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX6-NEXT: s_mov_b64 s[0:1], exec
+; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX6-NEXT: ; implicit-def: $vgpr1
; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX6-NEXT: s_cbranch_execz .LBB0_2
; GFX6-NEXT: ; %bb.1:
; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd
-; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX6-NEXT: s_mul_i32 s2, s2, 5
; GFX6-NEXT: v_mov_b32_e32 v1, s2
; GFX6-NEXT: v_mov_b32_e32 v2, 0
@@ -49,16 +49,16 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX8-LABEL: add_i32_constant:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB0_2
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX8-NEXT: s_mul_i32 s2, s2, 5
; GFX8-NEXT: v_mov_b32_e32 v1, s2
; GFX8-NEXT: v_mov_b32_e32 v2, 0
@@ -78,16 +78,16 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX9-LABEL: add_i32_constant:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB0_2
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_mul_i32 s2, s2, 5
; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: v_mov_b32_e32 v2, 0
@@ -106,18 +106,18 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX10W64-LABEL: add_i32_constant:
; GFX10W64: ; %bb.0: ; %entry
-; GFX10W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W64-NEXT: s_mov_b64 s[0:1], exec
; GFX10W64-NEXT: ; implicit-def: $vgpr1
-; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX10W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX10W64-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX10W64-NEXT: s_cbranch_execz .LBB0_2
; GFX10W64-NEXT: ; %bb.1:
; GFX10W64-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX10W64-NEXT: v_mov_b32_e32 v2, 0
; GFX10W64-NEXT: s_mul_i32 s2, s2, 5
+; GFX10W64-NEXT: v_mov_b32_e32 v2, 0
; GFX10W64-NEXT: v_mov_b32_e32 v1, s2
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
; GFX10W64-NEXT: buffer_atomic_add v1, v2, s[8:11], 0 idxen glc
@@ -135,17 +135,17 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX10W32-LABEL: add_i32_constant:
; GFX10W32: ; %bb.0: ; %entry
-; GFX10W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W32-NEXT: s_mov_b32 s0, exec_lo
; GFX10W32-NEXT: ; implicit-def: $vgpr1
-; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
+; GFX10W32-NEXT: s_bcnt1_i32_b32 s1, s0
; GFX10W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10W32-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX10W32-NEXT: s_cbranch_execz .LBB0_2
; GFX10W32-NEXT: ; %bb.1:
; GFX10W32-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX10W32-NEXT: v_mov_b32_e32 v2, 0
; GFX10W32-NEXT: s_mul_i32 s1, s1, 5
+; GFX10W32-NEXT: v_mov_b32_e32 v2, 0
; GFX10W32-NEXT: v_mov_b32_e32 v1, s1
; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX10W32-NEXT: buffer_atomic_add v1, v2, s[8:11], 0 idxen glc
@@ -163,20 +163,20 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX11W64-LABEL: add_i32_constant:
; GFX11W64: ; %bb.0: ; %entry
-; GFX11W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W64-NEXT: ; implicit-def: $vgpr1
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W64-NEXT: s_cbranch_execz .LBB0_2
; GFX11W64-NEXT: ; %bb.1:
; GFX11W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX11W64-NEXT: v_mov_b32_e32 v2, 0
; GFX11W64-NEXT: s_mul_i32 s2, s2, 5
-; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11W64-NEXT: v_mov_b32_e32 v2, 0
; GFX11W64-NEXT: v_mov_b32_e32 v1, s2
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W64-NEXT: buffer_atomic_add_u32 v1, v2, s[8:11], 0 idxen glc
@@ -194,20 +194,19 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX11W32-LABEL: add_i32_constant:
; GFX11W32: ; %bb.0: ; %entry
-; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX11W32-NEXT: ; implicit-def: $vgpr1
-; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX11W32-NEXT: s_mov_b32 s0, exec_lo
; GFX11W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W32-NEXT: s_cbranch_execz .LBB0_2
; GFX11W32-NEXT: ; %bb.1:
; GFX11W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX11W32-NEXT: v_mov_b32_e32 v2, 0
; GFX11W32-NEXT: s_mul_i32 s1, s1, 5
; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11W32-NEXT: v_mov_b32_e32 v1, s1
+; GFX11W32-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s1
; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W32-NEXT: buffer_atomic_add_u32 v1, v2, s[8:11], 0 idxen glc
; GFX11W32-NEXT: .LBB0_2:
@@ -224,21 +223,20 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX12W64-LABEL: add_i32_constant:
; GFX12W64: ; %bb.0: ; %entry
-; GFX12W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W64-NEXT: ; implicit-def: $vgpr1
-; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W64-NEXT: s_cbranch_execz .LBB0_2
; GFX12W64-NEXT: ; %bb.1:
; GFX12W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX12W64-NEXT: v_mov_b32_e32 v2, 0
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W64-NEXT: s_mul_i32 s2, s2, 5
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W64-NEXT: v_mov_b32_e32 v2, 0
; GFX12W64-NEXT: v_mov_b32_e32 v1, s2
; GFX12W64-NEXT: s_wait_kmcnt 0x0
; GFX12W64-NEXT: buffer_atomic_add_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN
@@ -257,19 +255,18 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX12W32-LABEL: add_i32_constant:
; GFX12W32: ; %bb.0: ; %entry
-; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX12W32-NEXT: ; implicit-def: $vgpr1
-; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX12W32-NEXT: s_mov_b32 s0, exec_lo
; GFX12W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W32-NEXT: s_cbranch_execz .LBB0_2
; GFX12W32-NEXT: ; %bb.1:
; GFX12W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W32-NEXT: s_mul_i32 s1, s1, 5
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12W32-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s1
; GFX12W32-NEXT: s_wait_kmcnt 0x0
; GFX12W32-NEXT: buffer_atomic_add_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN
@@ -287,20 +284,20 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX13W64-LABEL: add_i32_constant:
; GFX13W64: ; %bb.0: ; %entry
-; GFX13W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W64-NEXT: ; implicit-def: $vgpr1
-; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX13W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W64-NEXT: s_cbranch_execz .LBB0_2
; GFX13W64-NEXT: ; %bb.1:
; GFX13W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX13W64-NEXT: v_mov_b32_e32 v2, 0
; GFX13W64-NEXT: s_mul_i32 s2, s2, 5
-; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX13W64-NEXT: v_mov_b32_e32 v2, 0
; GFX13W64-NEXT: v_mov_b32_e32 v1, s2
; GFX13W64-NEXT: s_wait_kmcnt 0x0
; GFX13W64-NEXT: buffer_atomic_add_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN
@@ -318,18 +315,18 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX13W32-LABEL: add_i32_constant:
; GFX13W32: ; %bb.0: ; %entry
-; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX13W32-NEXT: ; implicit-def: $vgpr1
-; GFX13W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX13W32-NEXT: s_mov_b32 s0, exec_lo
; GFX13W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W32-NEXT: s_cbranch_execz .LBB0_2
; GFX13W32-NEXT: ; %bb.1:
; GFX13W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX13W32-NEXT: s_mul_i32 s1, s1, 5
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX13W32-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s1
; GFX13W32-NEXT: s_wait_kmcnt 0x0
; GFX13W32-NEXT: buffer_atomic_add_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN
@@ -353,27 +350,27 @@ entry:
define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(8) %inout, i32 %additive) {
; GFX6-LABEL: add_i32_uniform:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0x11
-; GFX6-NEXT: s_mov_b64 s[2:3], exec
-; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX6-NEXT: s_load_dword s2, s[4:5], 0x11
+; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX6-NEXT: s_mov_b64 s[0:1], exec
+; GFX6-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX6-NEXT: ; implicit-def: $vgpr1
; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX6-NEXT: s_cbranch_execz .LBB1_2
; GFX6-NEXT: ; %bb.1:
; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd
-; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mul_i32 s2, s6, s2
-; GFX6-NEXT: v_mov_b32_e32 v1, s2
+; GFX6-NEXT: s_mul_i32 s3, s2, s3
+; GFX6-NEXT: v_mov_b32_e32 v1, s3
; GFX6-NEXT: v_mov_b32_e32 v2, 0
; GFX6-NEXT: buffer_atomic_add v1, v2, s[8:11], 0 idxen glc
; GFX6-NEXT: .LBB1_2:
; GFX6-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX6-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_readfirstlane_b32 s4, v1
; GFX6-NEXT: s_mov_b32 s3, 0xf000
@@ -384,27 +381,27 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX8-LABEL: add_i32_uniform:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB1_2
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mul_i32 s2, s6, s2
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
+; GFX8-NEXT: s_mul_i32 s3, s2, s3
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: v_mov_b32_e32 v2, 0
; GFX8-NEXT: buffer_atomic_add v1, v2, s[8:11], 0 idxen glc
; GFX8-NEXT: .LBB1_2:
; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX8-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s2, v1
; GFX8-NEXT: v_add_u32_e32 v2, vcc, s2, v0
@@ -415,27 +412,27 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX9-LABEL: add_i32_uniform:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB1_2
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_i32 s2, s6, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: s_mul_i32 s3, s2, s3
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: buffer_atomic_add v1, v2, s[8:11], 0 idxen glc
; GFX9-NEXT: .LBB1_2:
; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX9-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s2, v1
; GFX9-NEXT: v_mov_b32_e32 v2, 0
@@ -445,30 +442,31 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX10W64-LABEL: add_i32_uniform:
; GFX10W64: ; %bb.0: ; %entry
-; GFX10W64-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX10W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX10W64-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W64-NEXT: s_mov_b64 s[0:1], exec
; GFX10W64-NEXT: ; implicit-def: $vgpr1
-; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX10W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX10W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX10W64-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX10W64-NEXT: s_cbranch_execz .LBB1_2
; GFX10W64-NEXT: ; %bb.1:
; GFX10W64-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX10W64-NEXT: v_mov_b32_e32 v2, 0
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10W64-NEXT: s_mul_i32 s2, s6, s2
-; GFX10W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX10W64-NEXT: s_mul_i32 s3, s2, s3
+; GFX10W64-NEXT: v_mov_b32_e32 v2, 0
+; GFX10W64-NEXT: v_mov_b32_e32 v1, s3
; GFX10W64-NEXT: buffer_atomic_add v1, v2, s[8:11], 0 idxen glc
; GFX10W64-NEXT: .LBB1_2:
; GFX10W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX10W64-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX10W64-NEXT: s_waitcnt vmcnt(0)
-; GFX10W64-NEXT: v_readfirstlane_b32 s2, v1
+; GFX10W64-NEXT: s_mov_b32 null, 0
+; GFX10W64-NEXT: v_readfirstlane_b32 s4, v1
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10W64-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s6, v0, s[2:3]
+; GFX10W64-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s2, v0, s[4:5]
; GFX10W64-NEXT: v_mov_b32_e32 v1, 0
; GFX10W64-NEXT: global_store_dword v1, v0, s[0:1]
; GFX10W64-NEXT: s_endpgm
@@ -476,18 +474,18 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX10W32-LABEL: add_i32_uniform:
; GFX10W32: ; %bb.0: ; %entry
; GFX10W32-NEXT: s_load_dword s0, s[4:5], 0x44
-; GFX10W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W32-NEXT: s_mov_b32 s1, exec_lo
; GFX10W32-NEXT: ; implicit-def: $vgpr1
-; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX10W32-NEXT: s_bcnt1_i32_b32 s2, s1
; GFX10W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10W32-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX10W32-NEXT: s_cbranch_execz .LBB1_2
; GFX10W32-NEXT: ; %bb.1:
; GFX10W32-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W32-NEXT: s_bcnt1_i32_b32 s2, s2
-; GFX10W32-NEXT: v_mov_b32_e32 v2, 0
; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX10W32-NEXT: s_mul_i32 s2, s0, s2
+; GFX10W32-NEXT: v_mov_b32_e32 v2, 0
; GFX10W32-NEXT: v_mov_b32_e32 v1, s2
; GFX10W32-NEXT: buffer_atomic_add v1, v2, s[8:11], 0 idxen glc
; GFX10W32-NEXT: .LBB1_2:
@@ -505,32 +503,32 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX11W64-LABEL: add_i32_uniform:
; GFX11W64: ; %bb.0: ; %entry
-; GFX11W64-NEXT: s_load_b32 s6, s[4:5], 0x44
-; GFX11W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX11W64-NEXT: s_load_b32 s2, s[4:5], 0x44
+; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W64-NEXT: ; implicit-def: $vgpr1
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W64-NEXT: s_cbranch_execz .LBB1_2
; GFX11W64-NEXT: ; %bb.1:
; GFX11W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX11W64-NEXT: v_mov_b32_e32 v2, 0
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11W64-NEXT: s_mul_i32 s2, s6, s2
-; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX11W64-NEXT: s_mul_i32 s3, s2, s3
+; GFX11W64-NEXT: v_mov_b32_e32 v2, 0
+; GFX11W64-NEXT: v_mov_b32_e32 v1, s3
; GFX11W64-NEXT: buffer_atomic_add_u32 v1, v2, s[8:11], 0 idxen glc
; GFX11W64-NEXT: .LBB1_2:
; GFX11W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX11W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX11W64-NEXT: s_waitcnt vmcnt(0)
-; GFX11W64-NEXT: v_readfirstlane_b32 s2, v1
+; GFX11W64-NEXT: v_readfirstlane_b32 s4, v1
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11W64-NEXT: v_mad_u64_u32 v[1:2], null, s6, v0, s[2:3]
+; GFX11W64-NEXT: v_mad_u64_u32 v[1:2], null, s2, v0, s[4:5]
; GFX11W64-NEXT: v_mov_b32_e32 v0, 0
; GFX11W64-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX11W64-NEXT: s_endpgm
@@ -538,21 +536,20 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX11W32-LABEL: add_i32_uniform:
; GFX11W32: ; %bb.0: ; %entry
; GFX11W32-NEXT: s_load_b32 s0, s[4:5], 0x44
-; GFX11W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W32-NEXT: ; implicit-def: $vgpr1
-; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
; GFX11W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W32-NEXT: s_cbranch_execz .LBB1_2
; GFX11W32-NEXT: ; %bb.1:
; GFX11W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W32-NEXT: s_bcnt1_i32_b32 s2, s2
-; GFX11W32-NEXT: v_mov_b32_e32 v2, 0
; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W32-NEXT: s_mul_i32 s2, s0, s2
; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11W32-NEXT: v_mov_b32_e32 v1, s2
+; GFX11W32-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
; GFX11W32-NEXT: buffer_atomic_add_u32 v1, v2, s[8:11], 0 idxen glc
; GFX11W32-NEXT: .LBB1_2:
; GFX11W32-NEXT: s_or_b32 exec_lo, exec_lo, s1
@@ -568,34 +565,32 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX12W64-LABEL: add_i32_uniform:
; GFX12W64: ; %bb.0: ; %entry
-; GFX12W64-NEXT: s_load_b32 s6, s[4:5], 0x44
-; GFX12W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX12W64-NEXT: s_load_b32 s2, s[4:5], 0x44
+; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W64-NEXT: ; implicit-def: $vgpr1
-; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W64-NEXT: s_cbranch_execz .LBB1_2
; GFX12W64-NEXT: ; %bb.1:
; GFX12W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX12W64-NEXT: v_mov_b32_e32 v2, 0
; GFX12W64-NEXT: s_wait_kmcnt 0x0
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12W64-NEXT: s_mul_i32 s2, s6, s2
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX12W64-NEXT: s_mul_i32 s3, s2, s3
+; GFX12W64-NEXT: v_mov_b32_e32 v2, 0
+; GFX12W64-NEXT: v_mov_b32_e32 v1, s3
; GFX12W64-NEXT: buffer_atomic_add_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN
; GFX12W64-NEXT: .LBB1_2:
; GFX12W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX12W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX12W64-NEXT: s_wait_loadcnt 0x0
-; GFX12W64-NEXT: v_readfirstlane_b32 s2, v1
+; GFX12W64-NEXT: v_readfirstlane_b32 s4, v1
; GFX12W64-NEXT: s_wait_kmcnt 0x0
-; GFX12W64-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12W64-NEXT: v_mad_co_u64_u32 v[0:1], null, s6, v0, s[2:3]
+; GFX12W64-NEXT: v_mad_co_u64_u32 v[0:1], null, s2, v0, s[4:5]
; GFX12W64-NEXT: v_mov_b32_e32 v1, 0
; GFX12W64-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX12W64-NEXT: s_endpgm
@@ -603,20 +598,19 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX12W32-LABEL: add_i32_uniform:
; GFX12W32: ; %bb.0: ; %entry
; GFX12W32-NEXT: s_load_b32 s0, s[4:5], 0x44
-; GFX12W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W32-NEXT: ; implicit-def: $vgpr1
-; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
; GFX12W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W32-NEXT: s_cbranch_execz .LBB1_2
; GFX12W32-NEXT: ; %bb.1:
; GFX12W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX12W32-NEXT: s_wait_kmcnt 0x0
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W32-NEXT: s_mul_i32 s2, s0, s2
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12W32-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
; GFX12W32-NEXT: buffer_atomic_add_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN
; GFX12W32-NEXT: .LBB1_2:
@@ -633,32 +627,32 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX13W64-LABEL: add_i32_uniform:
; GFX13W64: ; %bb.0: ; %entry
-; GFX13W64-NEXT: s_load_b32 s6, s[4:5], 0x44 nv
-; GFX13W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX13W64-NEXT: s_load_b32 s2, s[4:5], 0x44 nv
+; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W64-NEXT: ; implicit-def: $vgpr1
-; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX13W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W64-NEXT: s_cbranch_execz .LBB1_2
; GFX13W64-NEXT: ; %bb.1:
; GFX13W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX13W64-NEXT: v_mov_b32_e32 v2, 0
; GFX13W64-NEXT: s_wait_kmcnt 0x0
-; GFX13W64-NEXT: s_mul_i32 s2, s6, s2
-; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX13W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX13W64-NEXT: s_mul_i32 s3, s2, s3
+; GFX13W64-NEXT: v_mov_b32_e32 v2, 0
+; GFX13W64-NEXT: v_mov_b32_e32 v1, s3
; GFX13W64-NEXT: buffer_atomic_add_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN
; GFX13W64-NEXT: .LBB1_2:
; GFX13W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX13W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX13W64-NEXT: s_wait_loadcnt 0x0
-; GFX13W64-NEXT: v_readfirstlane_b32 s2, v1
+; GFX13W64-NEXT: v_readfirstlane_b32 s4, v1
; GFX13W64-NEXT: s_wait_kmcnt 0x0
; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13W64-NEXT: v_mad_co_u64_u32 v[0:1], null, s6, v0, s[2:3]
+; GFX13W64-NEXT: v_mad_co_u64_u32 v[0:1], null, s2, v0, s[4:5]
; GFX13W64-NEXT: v_mov_b32_e32 v1, 0
; GFX13W64-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX13W64-NEXT: s_endpgm
@@ -666,16 +660,16 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX13W32-LABEL: add_i32_uniform:
; GFX13W32: ; %bb.0: ; %entry
; GFX13W32-NEXT: s_load_b32 s0, s[4:5], 0x44 nv
-; GFX13W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W32-NEXT: ; implicit-def: $vgpr1
-; GFX13W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
; GFX13W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W32-NEXT: s_cbranch_execz .LBB1_2
; GFX13W32-NEXT: ; %bb.1:
; GFX13W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX13W32-NEXT: s_wait_kmcnt 0x0
; GFX13W32-NEXT: s_mul_i32 s2, s0, s2
; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -1450,16 +1444,16 @@ entry:
define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace(8) %inout) {
; GFX6-LABEL: sub_i32_constant:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_mov_b64 s[2:3], exec
-; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX6-NEXT: s_mov_b64 s[0:1], exec
+; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX6-NEXT: ; implicit-def: $vgpr1
; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX6-NEXT: s_cbranch_execz .LBB5_2
; GFX6-NEXT: ; %bb.1:
; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd
-; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX6-NEXT: s_mul_i32 s2, s2, 5
; GFX6-NEXT: v_mov_b32_e32 v1, s2
; GFX6-NEXT: v_mov_b32_e32 v2, 0
@@ -1480,16 +1474,16 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX8-LABEL: sub_i32_constant:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB5_2
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX8-NEXT: s_mul_i32 s2, s2, 5
; GFX8-NEXT: v_mov_b32_e32 v1, s2
; GFX8-NEXT: v_mov_b32_e32 v2, 0
@@ -1510,16 +1504,16 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX9-LABEL: sub_i32_constant:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB5_2
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_mul_i32 s2, s2, 5
; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: v_mov_b32_e32 v2, 0
@@ -1539,18 +1533,18 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX10W64-LABEL: sub_i32_constant:
; GFX10W64: ; %bb.0: ; %entry
-; GFX10W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W64-NEXT: s_mov_b64 s[0:1], exec
; GFX10W64-NEXT: ; implicit-def: $vgpr1
-; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX10W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX10W64-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX10W64-NEXT: s_cbranch_execz .LBB5_2
; GFX10W64-NEXT: ; %bb.1:
; GFX10W64-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX10W64-NEXT: v_mov_b32_e32 v2, 0
; GFX10W64-NEXT: s_mul_i32 s2, s2, 5
+; GFX10W64-NEXT: v_mov_b32_e32 v2, 0
; GFX10W64-NEXT: v_mov_b32_e32 v1, s2
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
; GFX10W64-NEXT: buffer_atomic_sub v1, v2, s[8:11], 0 idxen glc
@@ -1569,17 +1563,17 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX10W32-LABEL: sub_i32_constant:
; GFX10W32: ; %bb.0: ; %entry
-; GFX10W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W32-NEXT: s_mov_b32 s0, exec_lo
; GFX10W32-NEXT: ; implicit-def: $vgpr1
-; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
+; GFX10W32-NEXT: s_bcnt1_i32_b32 s1, s0
; GFX10W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10W32-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX10W32-NEXT: s_cbranch_execz .LBB5_2
; GFX10W32-NEXT: ; %bb.1:
; GFX10W32-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX10W32-NEXT: v_mov_b32_e32 v2, 0
; GFX10W32-NEXT: s_mul_i32 s1, s1, 5
+; GFX10W32-NEXT: v_mov_b32_e32 v2, 0
; GFX10W32-NEXT: v_mov_b32_e32 v1, s1
; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX10W32-NEXT: buffer_atomic_sub v1, v2, s[8:11], 0 idxen glc
@@ -1598,20 +1592,20 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX11W64-LABEL: sub_i32_constant:
; GFX11W64: ; %bb.0: ; %entry
-; GFX11W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W64-NEXT: ; implicit-def: $vgpr1
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W64-NEXT: s_cbranch_execz .LBB5_2
; GFX11W64-NEXT: ; %bb.1:
; GFX11W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX11W64-NEXT: v_mov_b32_e32 v2, 0
; GFX11W64-NEXT: s_mul_i32 s2, s2, 5
-; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11W64-NEXT: v_mov_b32_e32 v2, 0
; GFX11W64-NEXT: v_mov_b32_e32 v1, s2
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W64-NEXT: buffer_atomic_sub_u32 v1, v2, s[8:11], 0 idxen glc
@@ -1630,20 +1624,19 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX11W32-LABEL: sub_i32_constant:
; GFX11W32: ; %bb.0: ; %entry
-; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX11W32-NEXT: ; implicit-def: $vgpr1
-; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX11W32-NEXT: s_mov_b32 s0, exec_lo
; GFX11W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W32-NEXT: s_cbranch_execz .LBB5_2
; GFX11W32-NEXT: ; %bb.1:
; GFX11W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX11W32-NEXT: v_mov_b32_e32 v2, 0
; GFX11W32-NEXT: s_mul_i32 s1, s1, 5
; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11W32-NEXT: v_mov_b32_e32 v1, s1
+; GFX11W32-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s1
; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W32-NEXT: buffer_atomic_sub_u32 v1, v2, s[8:11], 0 idxen glc
; GFX11W32-NEXT: .LBB5_2:
@@ -1661,21 +1654,20 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX12W64-LABEL: sub_i32_constant:
; GFX12W64: ; %bb.0: ; %entry
-; GFX12W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W64-NEXT: ; implicit-def: $vgpr1
-; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W64-NEXT: s_cbranch_execz .LBB5_2
; GFX12W64-NEXT: ; %bb.1:
; GFX12W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX12W64-NEXT: v_mov_b32_e32 v2, 0
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W64-NEXT: s_mul_i32 s2, s2, 5
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W64-NEXT: v_mov_b32_e32 v2, 0
; GFX12W64-NEXT: v_mov_b32_e32 v1, s2
; GFX12W64-NEXT: s_wait_kmcnt 0x0
; GFX12W64-NEXT: buffer_atomic_sub_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN
@@ -1695,19 +1687,18 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX12W32-LABEL: sub_i32_constant:
; GFX12W32: ; %bb.0: ; %entry
-; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX12W32-NEXT: ; implicit-def: $vgpr1
-; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX12W32-NEXT: s_mov_b32 s0, exec_lo
; GFX12W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W32-NEXT: s_cbranch_execz .LBB5_2
; GFX12W32-NEXT: ; %bb.1:
; GFX12W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W32-NEXT: s_mul_i32 s1, s1, 5
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12W32-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s1
; GFX12W32-NEXT: s_wait_kmcnt 0x0
; GFX12W32-NEXT: buffer_atomic_sub_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN
@@ -1726,20 +1717,20 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX13W64-LABEL: sub_i32_constant:
; GFX13W64: ; %bb.0: ; %entry
-; GFX13W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W64-NEXT: ; implicit-def: $vgpr1
-; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX13W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W64-NEXT: s_cbranch_execz .LBB5_2
; GFX13W64-NEXT: ; %bb.1:
; GFX13W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX13W64-NEXT: v_mov_b32_e32 v2, 0
; GFX13W64-NEXT: s_mul_i32 s2, s2, 5
-; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX13W64-NEXT: v_mov_b32_e32 v2, 0
; GFX13W64-NEXT: v_mov_b32_e32 v1, s2
; GFX13W64-NEXT: s_wait_kmcnt 0x0
; GFX13W64-NEXT: buffer_atomic_sub_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN
@@ -1758,18 +1749,18 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX13W32-LABEL: sub_i32_constant:
; GFX13W32: ; %bb.0: ; %entry
-; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX13W32-NEXT: ; implicit-def: $vgpr1
-; GFX13W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX13W32-NEXT: s_mov_b32 s0, exec_lo
; GFX13W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W32-NEXT: s_cbranch_execz .LBB5_2
; GFX13W32-NEXT: ; %bb.1:
; GFX13W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX13W32-NEXT: s_mul_i32 s1, s1, 5
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX13W32-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s1
; GFX13W32-NEXT: s_wait_kmcnt 0x0
; GFX13W32-NEXT: buffer_atomic_sub_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN
@@ -1793,27 +1784,27 @@ entry:
define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(8) %inout, i32 %subitive) {
; GFX6-LABEL: sub_i32_uniform:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0x11
-; GFX6-NEXT: s_mov_b64 s[2:3], exec
-; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX6-NEXT: s_load_dword s2, s[4:5], 0x11
+; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX6-NEXT: s_mov_b64 s[0:1], exec
+; GFX6-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX6-NEXT: ; implicit-def: $vgpr1
; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX6-NEXT: s_cbranch_execz .LBB6_2
; GFX6-NEXT: ; %bb.1:
; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd
-; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mul_i32 s2, s6, s2
-; GFX6-NEXT: v_mov_b32_e32 v1, s2
+; GFX6-NEXT: s_mul_i32 s3, s2, s3
+; GFX6-NEXT: v_mov_b32_e32 v1, s3
; GFX6-NEXT: v_mov_b32_e32 v2, 0
; GFX6-NEXT: buffer_atomic_sub v1, v2, s[8:11], 0 idxen glc
; GFX6-NEXT: .LBB6_2:
; GFX6-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX6-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_readfirstlane_b32 s4, v1
; GFX6-NEXT: s_mov_b32 s3, 0xf000
@@ -1824,27 +1815,27 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX8-LABEL: sub_i32_uniform:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB6_2
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mul_i32 s2, s6, s2
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
+; GFX8-NEXT: s_mul_i32 s3, s2, s3
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: v_mov_b32_e32 v2, 0
; GFX8-NEXT: buffer_atomic_sub v1, v2, s[8:11], 0 idxen glc
; GFX8-NEXT: .LBB6_2:
; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX8-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s2, v1
; GFX8-NEXT: v_sub_u32_e32 v2, vcc, s2, v0
@@ -1855,27 +1846,27 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX9-LABEL: sub_i32_uniform:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB6_2
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_i32 s2, s6, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: s_mul_i32 s3, s2, s3
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: buffer_atomic_sub v1, v2, s[8:11], 0 idxen glc
; GFX9-NEXT: .LBB6_2:
; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX9-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s2, v1
; GFX9-NEXT: v_mov_b32_e32 v2, 0
@@ -1885,28 +1876,28 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX10W64-LABEL: sub_i32_uniform:
; GFX10W64: ; %bb.0: ; %entry
-; GFX10W64-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX10W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX10W64-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W64-NEXT: s_mov_b64 s[0:1], exec
; GFX10W64-NEXT: ; implicit-def: $vgpr1
-; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX10W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX10W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX10W64-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX10W64-NEXT: s_cbranch_execz .LBB6_2
; GFX10W64-NEXT: ; %bb.1:
; GFX10W64-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX10W64-NEXT: v_mov_b32_e32 v2, 0
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10W64-NEXT: s_mul_i32 s2, s6, s2
-; GFX10W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX10W64-NEXT: s_mul_i32 s3, s2, s3
+; GFX10W64-NEXT: v_mov_b32_e32 v2, 0
+; GFX10W64-NEXT: v_mov_b32_e32 v1, s3
; GFX10W64-NEXT: buffer_atomic_sub v1, v2, s[8:11], 0 idxen glc
; GFX10W64-NEXT: .LBB6_2:
; GFX10W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX10W64-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10W64-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX10W64-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX10W64-NEXT: s_waitcnt vmcnt(0)
; GFX10W64-NEXT: v_readfirstlane_b32 s2, v1
; GFX10W64-NEXT: v_mov_b32_e32 v1, 0
@@ -1917,18 +1908,18 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX10W32-LABEL: sub_i32_uniform:
; GFX10W32: ; %bb.0: ; %entry
; GFX10W32-NEXT: s_load_dword s0, s[4:5], 0x44
-; GFX10W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W32-NEXT: s_mov_b32 s1, exec_lo
; GFX10W32-NEXT: ; implicit-def: $vgpr1
-; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX10W32-NEXT: s_bcnt1_i32_b32 s2, s1
; GFX10W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10W32-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX10W32-NEXT: s_cbranch_execz .LBB6_2
; GFX10W32-NEXT: ; %bb.1:
; GFX10W32-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W32-NEXT: s_bcnt1_i32_b32 s2, s2
-; GFX10W32-NEXT: v_mov_b32_e32 v2, 0
; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX10W32-NEXT: s_mul_i32 s2, s0, s2
+; GFX10W32-NEXT: v_mov_b32_e32 v2, 0
; GFX10W32-NEXT: v_mov_b32_e32 v1, s2
; GFX10W32-NEXT: buffer_atomic_sub v1, v2, s[8:11], 0 idxen glc
; GFX10W32-NEXT: .LBB6_2:
@@ -1946,29 +1937,29 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX11W64-LABEL: sub_i32_uniform:
; GFX11W64: ; %bb.0: ; %entry
-; GFX11W64-NEXT: s_load_b32 s6, s[4:5], 0x44
-; GFX11W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX11W64-NEXT: s_load_b32 s2, s[4:5], 0x44
+; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W64-NEXT: ; implicit-def: $vgpr1
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W64-NEXT: s_cbranch_execz .LBB6_2
; GFX11W64-NEXT: ; %bb.1:
; GFX11W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX11W64-NEXT: v_mov_b32_e32 v2, 0
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11W64-NEXT: s_mul_i32 s2, s6, s2
-; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX11W64-NEXT: s_mul_i32 s3, s2, s3
+; GFX11W64-NEXT: v_mov_b32_e32 v2, 0
+; GFX11W64-NEXT: v_mov_b32_e32 v1, s3
; GFX11W64-NEXT: buffer_atomic_sub_u32 v1, v2, s[8:11], 0 idxen glc
; GFX11W64-NEXT: .LBB6_2:
; GFX11W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX11W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11W64-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX11W64-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX11W64-NEXT: s_waitcnt vmcnt(0)
; GFX11W64-NEXT: v_readfirstlane_b32 s2, v1
; GFX11W64-NEXT: v_mov_b32_e32 v1, 0
@@ -1980,21 +1971,20 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX11W32-LABEL: sub_i32_uniform:
; GFX11W32: ; %bb.0: ; %entry
; GFX11W32-NEXT: s_load_b32 s0, s[4:5], 0x44
-; GFX11W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W32-NEXT: ; implicit-def: $vgpr1
-; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
; GFX11W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W32-NEXT: s_cbranch_execz .LBB6_2
; GFX11W32-NEXT: ; %bb.1:
; GFX11W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W32-NEXT: s_bcnt1_i32_b32 s2, s2
-; GFX11W32-NEXT: v_mov_b32_e32 v2, 0
; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W32-NEXT: s_mul_i32 s2, s0, s2
; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11W32-NEXT: v_mov_b32_e32 v1, s2
+; GFX11W32-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
; GFX11W32-NEXT: buffer_atomic_sub_u32 v1, v2, s[8:11], 0 idxen glc
; GFX11W32-NEXT: .LBB6_2:
; GFX11W32-NEXT: s_or_b32 exec_lo, exec_lo, s1
@@ -2011,30 +2001,29 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX12W64-LABEL: sub_i32_uniform:
; GFX12W64: ; %bb.0: ; %entry
-; GFX12W64-NEXT: s_load_b32 s6, s[4:5], 0x44
-; GFX12W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX12W64-NEXT: s_load_b32 s2, s[4:5], 0x44
+; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W64-NEXT: ; implicit-def: $vgpr1
-; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W64-NEXT: s_cbranch_execz .LBB6_2
; GFX12W64-NEXT: ; %bb.1:
; GFX12W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX12W64-NEXT: v_mov_b32_e32 v2, 0
; GFX12W64-NEXT: s_wait_kmcnt 0x0
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12W64-NEXT: s_mul_i32 s2, s6, s2
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX12W64-NEXT: s_mul_i32 s3, s2, s3
+; GFX12W64-NEXT: v_mov_b32_e32 v2, 0
+; GFX12W64-NEXT: v_mov_b32_e32 v1, s3
; GFX12W64-NEXT: buffer_atomic_sub_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN
; GFX12W64-NEXT: .LBB6_2:
; GFX12W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX12W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX12W64-NEXT: s_wait_kmcnt 0x0
-; GFX12W64-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX12W64-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX12W64-NEXT: s_wait_loadcnt 0x0
; GFX12W64-NEXT: v_readfirstlane_b32 s2, v1
; GFX12W64-NEXT: v_mov_b32_e32 v1, 0
@@ -2047,20 +2036,19 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX12W32-LABEL: sub_i32_uniform:
; GFX12W32: ; %bb.0: ; %entry
; GFX12W32-NEXT: s_load_b32 s0, s[4:5], 0x44
-; GFX12W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W32-NEXT: ; implicit-def: $vgpr1
-; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
; GFX12W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W32-NEXT: s_cbranch_execz .LBB6_2
; GFX12W32-NEXT: ; %bb.1:
; GFX12W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX12W32-NEXT: s_wait_kmcnt 0x0
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W32-NEXT: s_mul_i32 s2, s0, s2
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12W32-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
; GFX12W32-NEXT: buffer_atomic_sub_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN
; GFX12W32-NEXT: .LBB6_2:
@@ -2079,29 +2067,29 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX13W64-LABEL: sub_i32_uniform:
; GFX13W64: ; %bb.0: ; %entry
-; GFX13W64-NEXT: s_load_b32 s6, s[4:5], 0x44 nv
-; GFX13W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX13W64-NEXT: s_load_b32 s2, s[4:5], 0x44 nv
+; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W64-NEXT: ; implicit-def: $vgpr1
-; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX13W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W64-NEXT: s_cbranch_execz .LBB6_2
; GFX13W64-NEXT: ; %bb.1:
; GFX13W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX13W64-NEXT: v_mov_b32_e32 v2, 0
; GFX13W64-NEXT: s_wait_kmcnt 0x0
-; GFX13W64-NEXT: s_mul_i32 s2, s6, s2
-; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX13W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX13W64-NEXT: s_mul_i32 s3, s2, s3
+; GFX13W64-NEXT: v_mov_b32_e32 v2, 0
+; GFX13W64-NEXT: v_mov_b32_e32 v1, s3
; GFX13W64-NEXT: buffer_atomic_sub_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN
; GFX13W64-NEXT: .LBB6_2:
; GFX13W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX13W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX13W64-NEXT: s_wait_kmcnt 0x0
-; GFX13W64-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX13W64-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX13W64-NEXT: s_wait_loadcnt 0x0
; GFX13W64-NEXT: v_readfirstlane_b32 s2, v1
; GFX13W64-NEXT: v_mov_b32_e32 v1, 0
@@ -2113,16 +2101,16 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX13W32-LABEL: sub_i32_uniform:
; GFX13W32: ; %bb.0: ; %entry
; GFX13W32-NEXT: s_load_b32 s0, s[4:5], 0x44 nv
-; GFX13W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W32-NEXT: ; implicit-def: $vgpr1
-; GFX13W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
; GFX13W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W32-NEXT: s_cbranch_execz .LBB6_2
; GFX13W32-NEXT: ; %bb.1:
; GFX13W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX13W32-NEXT: s_wait_kmcnt 0x0
; GFX13W32-NEXT: s_mul_i32 s2, s0, s2
; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
diff --git a/llvm/test/CodeGen/AMDGPU/atomics-hw-remarks-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/atomics-hw-remarks-gfx90a.ll
index ca7cc989c510c..fd91f43ac3868 100644
--- a/llvm/test/CodeGen/AMDGPU/atomics-hw-remarks-gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomics-hw-remarks-gfx90a.ll
@@ -11,7 +11,7 @@
; GFX90A-HW: Hardware instruction generated for atomic fadd operation at memory scope singlethread-one-as due to an unsafe request.
; GFX90A-HW-LABEL: atomic_add_unsafe_hw:
-; GFX90A-HW: ds_add_f64 v2, v[0:1]
+; GFX90A-HW: ds_add_f64 v0, v[2:3]
; GFX90A-HW: s_endpgm
define amdgpu_kernel void @atomic_add_unsafe_hw(ptr addrspace(3) %ptr) #0 {
main_body:
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-no-rtn.ll b/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-no-rtn.ll
index 8a015cf3e41e6..b41acb95772a5 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-no-rtn.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-no-rtn.ll
@@ -58,41 +58,24 @@ define amdgpu_ps void @global_atomic_fadd_f32_saddr_no_rtn_atomicrmw(ptr addrspa
; GFX908-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
; GFX908-NEXT: [[V_MBCNT_LO_U32_B32_e64_:%[0-9]+]]:vgpr_32 = V_MBCNT_LO_U32_B32_e64 killed [[COPY6]], [[COPY7]], implicit $exec
; GFX908-NEXT: [[V_MBCNT_HI_U32_B32_e64_:%[0-9]+]]:vgpr_32 = V_MBCNT_HI_U32_B32_e64 killed [[COPY5]], killed [[V_MBCNT_LO_U32_B32_e64_]], implicit $exec
- ; GFX908-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -2147483648, implicit $exec
- ; GFX908-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
- ; GFX908-NEXT: [[V_SET_INACTIVE_B32_:%[0-9]+]]:vgpr_32 = V_SET_INACTIVE_B32 0, [[COPY]], 0, [[V_MOV_B32_e32_]], killed [[DEF]], implicit $exec
- ; GFX908-NEXT: [[V_MOV_B32_dpp:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[V_MOV_B32_e32_]], [[V_SET_INACTIVE_B32_]], 273, 15, 15, 0, implicit $exec
- ; GFX908-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_SET_INACTIVE_B32_]], 0, killed [[V_MOV_B32_dpp]], 0, 0, implicit $mode, implicit $exec
- ; GFX908-NEXT: [[V_MOV_B32_dpp1:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[V_MOV_B32_e32_]], [[V_ADD_F32_e64_]], 274, 15, 15, 0, implicit $exec
- ; GFX908-NEXT: [[V_ADD_F32_e64_1:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_]], 0, killed [[V_MOV_B32_dpp1]], 0, 0, implicit $mode, implicit $exec
- ; GFX908-NEXT: [[V_MOV_B32_dpp2:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[V_MOV_B32_e32_]], [[V_ADD_F32_e64_1]], 276, 15, 15, 0, implicit $exec
- ; GFX908-NEXT: [[V_ADD_F32_e64_2:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_1]], 0, killed [[V_MOV_B32_dpp2]], 0, 0, implicit $mode, implicit $exec
- ; GFX908-NEXT: [[V_MOV_B32_dpp3:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[V_MOV_B32_e32_]], [[V_ADD_F32_e64_2]], 280, 15, 15, 0, implicit $exec
- ; GFX908-NEXT: [[V_ADD_F32_e64_3:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_2]], 0, killed [[V_MOV_B32_dpp3]], 0, 0, implicit $mode, implicit $exec
- ; GFX908-NEXT: [[V_MOV_B32_dpp4:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[V_MOV_B32_e32_]], [[V_ADD_F32_e64_3]], 322, 10, 15, 0, implicit $exec
- ; GFX908-NEXT: [[V_ADD_F32_e64_4:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_3]], 0, killed [[V_MOV_B32_dpp4]], 0, 0, implicit $mode, implicit $exec
- ; GFX908-NEXT: [[V_MOV_B32_dpp5:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[V_MOV_B32_e32_]], [[V_ADD_F32_e64_4]], 323, 12, 15, 0, implicit $exec
- ; GFX908-NEXT: [[V_ADD_F32_e64_5:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_4]], 0, killed [[V_MOV_B32_dpp5]], 0, 0, implicit $mode, implicit $exec
- ; GFX908-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 63
- ; GFX908-NEXT: [[V_READLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READLANE_B32 killed [[V_ADD_F32_e64_5]], killed [[S_MOV_B32_1]]
- ; GFX908-NEXT: early-clobber %1:sgpr_32 = STRICT_WWM killed [[V_READLANE_B32_]], implicit $exec
+ ; GFX908-NEXT: [[WAVE_REDUCE_FADD_PSEUDO_F32_:%[0-9]+]]:sgpr_32 = WAVE_REDUCE_FADD_PSEUDO_F32 [[COPY]], 2, implicit $exec
+ ; GFX908-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[WAVE_REDUCE_FADD_PSEUDO_F32_]]
; GFX908-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 killed [[V_MBCNT_HI_U32_B32_e64_]], [[S_MOV_B32_]], implicit $exec
; GFX908-NEXT: [[SI_IF1:%[0-9]+]]:sreg_64 = SI_IF killed [[V_CMP_EQ_U32_e64_]], %bb.3, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; GFX908-NEXT: S_BRANCH %bb.2
; GFX908-NEXT: {{ $}}
- ; GFX908-NEXT: bb.2 (%ir-block.31):
+ ; GFX908-NEXT: bb.2 (%ir-block.17):
; GFX908-NEXT: successors: %bb.3(0x80000000)
; GFX908-NEXT: {{ $}}
- ; GFX908-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; GFX908-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY %1
- ; GFX908-NEXT: GLOBAL_ATOMIC_ADD_F32_SADDR killed [[V_MOV_B32_e32_1]], [[COPY8]], [[COPY3]], 0, 0, implicit $exec :: (load store syncscope("wavefront") monotonic (s32) on %ir.ptr, addrspace 1)
+ ; GFX908-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX908-NEXT: GLOBAL_ATOMIC_ADD_F32_SADDR killed [[V_MOV_B32_e32_]], [[COPY8]], [[COPY3]], 0, 0, implicit $exec :: (load store syncscope("wavefront") monotonic (s32) on %ir.ptr, addrspace 1)
; GFX908-NEXT: {{ $}}
; GFX908-NEXT: bb.3.Flow:
; GFX908-NEXT: successors: %bb.4(0x80000000)
; GFX908-NEXT: {{ $}}
; GFX908-NEXT: SI_END_CF [[SI_IF1]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; GFX908-NEXT: {{ $}}
- ; GFX908-NEXT: bb.4 (%ir-block.33):
+ ; GFX908-NEXT: bb.4 (%ir-block.19):
; GFX908-NEXT: SI_END_CF [[SI_IF]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; GFX908-NEXT: S_ENDPGM 0
;
@@ -120,41 +103,24 @@ define amdgpu_ps void @global_atomic_fadd_f32_saddr_no_rtn_atomicrmw(ptr addrspa
; GFX90A_GFX942-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
; GFX90A_GFX942-NEXT: [[V_MBCNT_LO_U32_B32_e64_:%[0-9]+]]:vgpr_32 = V_MBCNT_LO_U32_B32_e64 killed [[COPY6]], [[COPY7]], implicit $exec
; GFX90A_GFX942-NEXT: [[V_MBCNT_HI_U32_B32_e64_:%[0-9]+]]:vgpr_32 = V_MBCNT_HI_U32_B32_e64 killed [[COPY5]], killed [[V_MBCNT_LO_U32_B32_e64_]], implicit $exec
- ; GFX90A_GFX942-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -2147483648, implicit $exec
- ; GFX90A_GFX942-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
- ; GFX90A_GFX942-NEXT: [[V_SET_INACTIVE_B32_:%[0-9]+]]:vgpr_32 = V_SET_INACTIVE_B32 0, [[COPY]], 0, [[V_MOV_B32_e32_]], killed [[DEF]], implicit $exec
- ; GFX90A_GFX942-NEXT: [[V_MOV_B32_dpp:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[V_MOV_B32_e32_]], [[V_SET_INACTIVE_B32_]], 273, 15, 15, 0, implicit $exec
- ; GFX90A_GFX942-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_SET_INACTIVE_B32_]], 0, killed [[V_MOV_B32_dpp]], 0, 0, implicit $mode, implicit $exec
- ; GFX90A_GFX942-NEXT: [[V_MOV_B32_dpp1:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[V_MOV_B32_e32_]], [[V_ADD_F32_e64_]], 274, 15, 15, 0, implicit $exec
- ; GFX90A_GFX942-NEXT: [[V_ADD_F32_e64_1:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_]], 0, killed [[V_MOV_B32_dpp1]], 0, 0, implicit $mode, implicit $exec
- ; GFX90A_GFX942-NEXT: [[V_MOV_B32_dpp2:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[V_MOV_B32_e32_]], [[V_ADD_F32_e64_1]], 276, 15, 15, 0, implicit $exec
- ; GFX90A_GFX942-NEXT: [[V_ADD_F32_e64_2:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_1]], 0, killed [[V_MOV_B32_dpp2]], 0, 0, implicit $mode, implicit $exec
- ; GFX90A_GFX942-NEXT: [[V_MOV_B32_dpp3:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[V_MOV_B32_e32_]], [[V_ADD_F32_e64_2]], 280, 15, 15, 0, implicit $exec
- ; GFX90A_GFX942-NEXT: [[V_ADD_F32_e64_3:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_2]], 0, killed [[V_MOV_B32_dpp3]], 0, 0, implicit $mode, implicit $exec
- ; GFX90A_GFX942-NEXT: [[V_MOV_B32_dpp4:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[V_MOV_B32_e32_]], [[V_ADD_F32_e64_3]], 322, 10, 15, 0, implicit $exec
- ; GFX90A_GFX942-NEXT: [[V_ADD_F32_e64_4:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_3]], 0, killed [[V_MOV_B32_dpp4]], 0, 0, implicit $mode, implicit $exec
- ; GFX90A_GFX942-NEXT: [[V_MOV_B32_dpp5:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[V_MOV_B32_e32_]], [[V_ADD_F32_e64_4]], 323, 12, 15, 0, implicit $exec
- ; GFX90A_GFX942-NEXT: [[V_ADD_F32_e64_5:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_4]], 0, killed [[V_MOV_B32_dpp5]], 0, 0, implicit $mode, implicit $exec
- ; GFX90A_GFX942-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 63
- ; GFX90A_GFX942-NEXT: [[V_READLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READLANE_B32 killed [[V_ADD_F32_e64_5]], killed [[S_MOV_B32_1]]
- ; GFX90A_GFX942-NEXT: early-clobber %1:sgpr_32 = STRICT_WWM killed [[V_READLANE_B32_]], implicit $exec
+ ; GFX90A_GFX942-NEXT: [[WAVE_REDUCE_FADD_PSEUDO_F32_:%[0-9]+]]:sgpr_32 = WAVE_REDUCE_FADD_PSEUDO_F32 [[COPY]], 2, implicit $exec
+ ; GFX90A_GFX942-NEXT: [[COPY8:%[0-9]+]]:av_32 = COPY [[WAVE_REDUCE_FADD_PSEUDO_F32_]]
; GFX90A_GFX942-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 killed [[V_MBCNT_HI_U32_B32_e64_]], [[S_MOV_B32_]], implicit $exec
; GFX90A_GFX942-NEXT: [[SI_IF1:%[0-9]+]]:sreg_64 = SI_IF killed [[V_CMP_EQ_U32_e64_]], %bb.3, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; GFX90A_GFX942-NEXT: S_BRANCH %bb.2
; GFX90A_GFX942-NEXT: {{ $}}
- ; GFX90A_GFX942-NEXT: bb.2 (%ir-block.31):
+ ; GFX90A_GFX942-NEXT: bb.2 (%ir-block.17):
; GFX90A_GFX942-NEXT: successors: %bb.3(0x80000000)
; GFX90A_GFX942-NEXT: {{ $}}
- ; GFX90A_GFX942-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; GFX90A_GFX942-NEXT: [[COPY8:%[0-9]+]]:av_32 = COPY %1
- ; GFX90A_GFX942-NEXT: GLOBAL_ATOMIC_ADD_F32_SADDR killed [[V_MOV_B32_e32_1]], [[COPY8]], [[COPY3]], 0, 0, implicit $exec :: (load store syncscope("wavefront") monotonic (s32) on %ir.ptr, addrspace 1)
+ ; GFX90A_GFX942-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX90A_GFX942-NEXT: GLOBAL_ATOMIC_ADD_F32_SADDR killed [[V_MOV_B32_e32_]], [[COPY8]], [[COPY3]], 0, 0, implicit $exec :: (load store syncscope("wavefront") monotonic (s32) on %ir.ptr, addrspace 1)
; GFX90A_GFX942-NEXT: {{ $}}
; GFX90A_GFX942-NEXT: bb.3.Flow:
; GFX90A_GFX942-NEXT: successors: %bb.4(0x80000000)
; GFX90A_GFX942-NEXT: {{ $}}
; GFX90A_GFX942-NEXT: SI_END_CF [[SI_IF1]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; GFX90A_GFX942-NEXT: {{ $}}
- ; GFX90A_GFX942-NEXT: bb.4 (%ir-block.33):
+ ; GFX90A_GFX942-NEXT: bb.4 (%ir-block.19):
; GFX90A_GFX942-NEXT: SI_END_CF [[SI_IF]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; GFX90A_GFX942-NEXT: S_ENDPGM 0
;
@@ -178,38 +144,24 @@ define amdgpu_ps void @global_atomic_fadd_f32_saddr_no_rtn_atomicrmw(ptr addrspa
; GFX11_GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $exec_lo
; GFX11_GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GFX11_GFX12-NEXT: [[V_MBCNT_LO_U32_B32_e64_:%[0-9]+]]:vgpr_32 = V_MBCNT_LO_U32_B32_e64 [[COPY4]], [[S_MOV_B32_]], implicit $exec
- ; GFX11_GFX12-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -2147483648, implicit $exec
- ; GFX11_GFX12-NEXT: [[DEF:%[0-9]+]]:sreg_32_xm0_xexec = IMPLICIT_DEF
- ; GFX11_GFX12-NEXT: [[V_SET_INACTIVE_B32_:%[0-9]+]]:vgpr_32 = V_SET_INACTIVE_B32 0, [[COPY]], 0, [[V_MOV_B32_e32_]], killed [[DEF]], implicit $exec
- ; GFX11_GFX12-NEXT: [[V_MOV_B32_dpp:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[V_MOV_B32_e32_]], [[V_SET_INACTIVE_B32_]], 353, 15, 15, 0, implicit $exec
- ; GFX11_GFX12-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_SET_INACTIVE_B32_]], 0, killed [[V_MOV_B32_dpp]], 0, 0, implicit $mode, implicit $exec
- ; GFX11_GFX12-NEXT: [[V_MOV_B32_dpp1:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[V_MOV_B32_e32_]], [[V_ADD_F32_e64_]], 354, 15, 15, 0, implicit $exec
- ; GFX11_GFX12-NEXT: [[V_ADD_F32_e64_1:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_]], 0, killed [[V_MOV_B32_dpp1]], 0, 0, implicit $mode, implicit $exec
- ; GFX11_GFX12-NEXT: [[V_MOV_B32_dpp2:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[V_MOV_B32_e32_]], [[V_ADD_F32_e64_1]], 356, 15, 15, 0, implicit $exec
- ; GFX11_GFX12-NEXT: [[V_ADD_F32_e64_2:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_1]], 0, killed [[V_MOV_B32_dpp2]], 0, 0, implicit $mode, implicit $exec
- ; GFX11_GFX12-NEXT: [[V_MOV_B32_dpp3:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[V_MOV_B32_e32_]], [[V_ADD_F32_e64_2]], 360, 15, 15, 0, implicit $exec
- ; GFX11_GFX12-NEXT: [[V_ADD_F32_e64_3:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_2]], 0, killed [[V_MOV_B32_dpp3]], 0, 0, implicit $mode, implicit $exec
- ; GFX11_GFX12-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; GFX11_GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[DEF1]]
- ; GFX11_GFX12-NEXT: [[V_PERMLANEX16_B32_e64_:%[0-9]+]]:vgpr_32 = V_PERMLANEX16_B32_e64 0, [[V_ADD_F32_e64_3]], 0, [[S_MOV_B32_]], 0, [[S_MOV_B32_]], [[COPY5]], 0, implicit $exec
- ; GFX11_GFX12-NEXT: [[V_ADD_F32_e64_4:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_3]], 0, killed [[V_PERMLANEX16_B32_e64_]], 0, 0, implicit $mode, implicit $exec
- ; GFX11_GFX12-NEXT: early-clobber %1:vgpr_32 = STRICT_WWM killed [[V_ADD_F32_e64_4]], implicit $exec
+ ; GFX11_GFX12-NEXT: [[WAVE_REDUCE_FADD_PSEUDO_F32_:%[0-9]+]]:sgpr_32 = WAVE_REDUCE_FADD_PSEUDO_F32 [[COPY]], 2, implicit $exec
+ ; GFX11_GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[WAVE_REDUCE_FADD_PSEUDO_F32_]]
; GFX11_GFX12-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_EQ_U32_e64 killed [[V_MBCNT_LO_U32_B32_e64_]], [[S_MOV_B32_]], implicit $exec
; GFX11_GFX12-NEXT: [[SI_IF1:%[0-9]+]]:sreg_32 = SI_IF killed [[V_CMP_EQ_U32_e64_]], %bb.3, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; GFX11_GFX12-NEXT: S_BRANCH %bb.2
; GFX11_GFX12-NEXT: {{ $}}
- ; GFX11_GFX12-NEXT: bb.2 (%ir-block.24):
+ ; GFX11_GFX12-NEXT: bb.2 (%ir-block.13):
; GFX11_GFX12-NEXT: successors: %bb.3(0x80000000)
; GFX11_GFX12-NEXT: {{ $}}
- ; GFX11_GFX12-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; GFX11_GFX12-NEXT: GLOBAL_ATOMIC_ADD_F32_SADDR killed [[V_MOV_B32_e32_1]], %1, [[COPY3]], 0, 0, implicit $exec :: (load store syncscope("wavefront") monotonic (s32) on %ir.ptr, addrspace 1)
+ ; GFX11_GFX12-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX11_GFX12-NEXT: GLOBAL_ATOMIC_ADD_F32_SADDR killed [[V_MOV_B32_e32_]], [[COPY5]], [[COPY3]], 0, 0, implicit $exec :: (load store syncscope("wavefront") monotonic (s32) on %ir.ptr, addrspace 1)
; GFX11_GFX12-NEXT: {{ $}}
; GFX11_GFX12-NEXT: bb.3.Flow:
; GFX11_GFX12-NEXT: successors: %bb.4(0x80000000)
; GFX11_GFX12-NEXT: {{ $}}
; GFX11_GFX12-NEXT: SI_END_CF [[SI_IF1]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; GFX11_GFX12-NEXT: {{ $}}
- ; GFX11_GFX12-NEXT: bb.4 (%ir-block.26):
+ ; GFX11_GFX12-NEXT: bb.4 (%ir-block.15):
; GFX11_GFX12-NEXT: SI_END_CF [[SI_IF]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; GFX11_GFX12-NEXT: S_ENDPGM 0
%ret = atomicrmw fadd ptr addrspace(1) %ptr, float %data syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !0
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomic-scan.ll b/llvm/test/CodeGen/AMDGPU/global-atomic-scan.ll
index 4c90b21d0899c..c9469d393d2ea 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomic-scan.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomic-scan.ll
@@ -11,15 +11,13 @@ define amdgpu_kernel void @atomic_add_i32_offset(ptr addrspace(1) %out, i32 %in)
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile add ptr addrspace(1) [[GEP]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -38,15 +36,13 @@ define amdgpu_kernel void @atomic_add_i32_max_neg_offset(ptr addrspace(1) %out,
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile add ptr addrspace(1) [[GEP]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -65,15 +61,13 @@ define amdgpu_kernel void @atomic_add_i32_soffset(ptr addrspace(1) %out, i32 %in
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile add ptr addrspace(1) [[GEP]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -92,15 +86,13 @@ define amdgpu_kernel void @atomic_add_i32_huge_offset(ptr addrspace(1) %out, i32
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile add ptr addrspace(1) [[GEP]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -120,15 +112,13 @@ define amdgpu_kernel void @atomic_add_i32_ret_offset(ptr addrspace(1) %out, ptr
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile add ptr addrspace(1) [[GEP]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: [[TMP13:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP11]], [[TMP10]] ]
; IR-NEXT: [[TMP14:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP13]])
; IR-NEXT: [[TMP15:%.*]] = mul i32 [[IN]], [[TMP5]]
@@ -154,15 +144,13 @@ define amdgpu_kernel void @atomic_add_i32_addr64_offset(ptr addrspace(1) %out, i
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile add ptr addrspace(1) [[GEP]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -183,15 +171,13 @@ define amdgpu_kernel void @atomic_add_i32_ret_addr64_offset(ptr addrspace(1) %ou
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile add ptr addrspace(1) [[GEP]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: [[TMP13:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP11]], [[TMP10]] ]
; IR-NEXT: [[TMP14:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP13]])
; IR-NEXT: [[TMP15:%.*]] = mul i32 [[IN]], [[TMP5]]
@@ -216,15 +202,13 @@ define amdgpu_kernel void @atomic_add_i32(ptr addrspace(1) %out, i32 %in) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile add ptr addrspace(1) [[OUT:%.*]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -241,15 +225,13 @@ define amdgpu_kernel void @atomic_add_i32_ret(ptr addrspace(1) %out, ptr addrspa
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile add ptr addrspace(1) [[OUT:%.*]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: [[TMP13:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP11]], [[TMP10]] ]
; IR-NEXT: [[TMP14:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP13]])
; IR-NEXT: [[TMP15:%.*]] = mul i32 [[IN]], [[TMP5]]
@@ -273,15 +255,13 @@ define amdgpu_kernel void @atomic_add_i32_addr64(ptr addrspace(1) %out, i32 %in,
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile add ptr addrspace(1) [[PTR]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -300,15 +280,13 @@ define amdgpu_kernel void @atomic_add_i32_ret_addr64(ptr addrspace(1) %out, ptr
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile add ptr addrspace(1) [[PTR]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: [[TMP13:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP11]], [[TMP10]] ]
; IR-NEXT: [[TMP14:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP13]])
; IR-NEXT: [[TMP15:%.*]] = mul i32 [[IN]], [[TMP5]]
@@ -333,12 +311,13 @@ define amdgpu_kernel void @atomic_and_i32_offset(ptr addrspace(1) %out, i32 %in)
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.and.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile and ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile and ptr addrspace(1) [[GEP]], i32 [[IN]] seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -357,15 +336,16 @@ define amdgpu_kernel void @atomic_and_i32_ret_offset(ptr addrspace(1) %out, ptr
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.and.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile and ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile and ptr addrspace(1) [[GEP]], i32 [[IN]] seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ]
; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP10]])
-; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -1, i32 [[IN]]
+; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -1, i32 [[IN1]]
; IR-NEXT: [[TMP13:%.*]] = and i32 [[TMP11]], [[TMP12]]
; IR-NEXT: store i32 [[TMP13]], ptr addrspace(1) [[OUT2:%.*]], align 4
; IR-NEXT: ret void
@@ -388,12 +368,13 @@ define amdgpu_kernel void @atomic_and_i32_addr64_offset(ptr addrspace(1) %out, i
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.and.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile and ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile and ptr addrspace(1) [[GEP]], i32 [[IN]] seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -414,15 +395,16 @@ define amdgpu_kernel void @atomic_and_i32_ret_addr64_offset(ptr addrspace(1) %ou
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.and.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile and ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile and ptr addrspace(1) [[GEP]], i32 [[IN]] seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ]
; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP10]])
-; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -1, i32 [[IN]]
+; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -1, i32 [[IN1]]
; IR-NEXT: [[TMP13:%.*]] = and i32 [[TMP11]], [[TMP12]]
; IR-NEXT: store i32 [[TMP13]], ptr addrspace(1) [[OUT2:%.*]], align 4
; IR-NEXT: ret void
@@ -444,12 +426,13 @@ define amdgpu_kernel void @atomic_and_i32(ptr addrspace(1) %out, i32 %in) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.and.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile and ptr addrspace(1) [[OUT:%.*]], i32 [[IN:%.*]] seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile and ptr addrspace(1) [[OUT:%.*]], i32 [[IN]] seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -466,15 +449,16 @@ define amdgpu_kernel void @atomic_and_i32_ret(ptr addrspace(1) %out, ptr addrspa
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.and.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile and ptr addrspace(1) [[OUT:%.*]], i32 [[IN:%.*]] seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile and ptr addrspace(1) [[OUT:%.*]], i32 [[IN]] seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ]
; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP10]])
-; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -1, i32 [[IN]]
+; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -1, i32 [[IN1]]
; IR-NEXT: [[TMP13:%.*]] = and i32 [[TMP11]], [[TMP12]]
; IR-NEXT: store i32 [[TMP13]], ptr addrspace(1) [[OUT2:%.*]], align 4
; IR-NEXT: ret void
@@ -495,12 +479,13 @@ define amdgpu_kernel void @atomic_and_i32_addr64(ptr addrspace(1) %out, i32 %in,
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.and.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile and ptr addrspace(1) [[PTR]], i32 [[IN:%.*]] seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile and ptr addrspace(1) [[PTR]], i32 [[IN]] seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -519,15 +504,16 @@ define amdgpu_kernel void @atomic_and_i32_ret_addr64(ptr addrspace(1) %out, ptr
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.and.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile and ptr addrspace(1) [[PTR]], i32 [[IN:%.*]] seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile and ptr addrspace(1) [[PTR]], i32 [[IN]] seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ]
; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP10]])
-; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -1, i32 [[IN]]
+; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -1, i32 [[IN1]]
; IR-NEXT: [[TMP13:%.*]] = and i32 [[TMP11]], [[TMP12]]
; IR-NEXT: store i32 [[TMP13]], ptr addrspace(1) [[OUT2:%.*]], align 4
; IR-NEXT: ret void
@@ -549,15 +535,13 @@ define amdgpu_kernel void @atomic_sub_i32_offset(ptr addrspace(1) %out, i32 %in)
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile sub ptr addrspace(1) [[GEP]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -576,15 +560,13 @@ define amdgpu_kernel void @atomic_sub_i32_ret_offset(ptr addrspace(1) %out, ptr
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile sub ptr addrspace(1) [[GEP]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: [[TMP13:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP11]], [[TMP10]] ]
; IR-NEXT: [[TMP14:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP13]])
; IR-NEXT: [[TMP15:%.*]] = mul i32 [[IN]], [[TMP5]]
@@ -610,15 +592,13 @@ define amdgpu_kernel void @atomic_sub_i32_addr64_offset(ptr addrspace(1) %out, i
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile sub ptr addrspace(1) [[GEP]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -639,15 +619,13 @@ define amdgpu_kernel void @atomic_sub_i32_ret_addr64_offset(ptr addrspace(1) %ou
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile sub ptr addrspace(1) [[GEP]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: [[TMP13:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP11]], [[TMP10]] ]
; IR-NEXT: [[TMP14:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP13]])
; IR-NEXT: [[TMP15:%.*]] = mul i32 [[IN]], [[TMP5]]
@@ -672,15 +650,13 @@ define amdgpu_kernel void @atomic_sub_i32(ptr addrspace(1) %out, i32 %in) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile sub ptr addrspace(1) [[OUT:%.*]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -697,15 +673,13 @@ define amdgpu_kernel void @atomic_sub_i32_ret(ptr addrspace(1) %out, ptr addrspa
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile sub ptr addrspace(1) [[OUT:%.*]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: [[TMP13:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP11]], [[TMP10]] ]
; IR-NEXT: [[TMP14:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP13]])
; IR-NEXT: [[TMP15:%.*]] = mul i32 [[IN]], [[TMP5]]
@@ -729,15 +703,13 @@ define amdgpu_kernel void @atomic_sub_i32_addr64(ptr addrspace(1) %out, i32 %in,
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile sub ptr addrspace(1) [[PTR]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -756,15 +728,13 @@ define amdgpu_kernel void @atomic_sub_i32_ret_addr64(ptr addrspace(1) %out, ptr
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile sub ptr addrspace(1) [[PTR]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: [[TMP13:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP11]], [[TMP10]] ]
; IR-NEXT: [[TMP14:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP13]])
; IR-NEXT: [[TMP15:%.*]] = mul i32 [[IN]], [[TMP5]]
@@ -789,12 +759,13 @@ define amdgpu_kernel void @atomic_max_i32_offset(ptr addrspace(1) %out, i32 %in)
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.max.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile max ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile max ptr addrspace(1) [[GEP]], i32 [[IN]] seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -813,15 +784,16 @@ define amdgpu_kernel void @atomic_max_i32_ret_offset(ptr addrspace(1) %out, ptr
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.max.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile max ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile max ptr addrspace(1) [[GEP]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ]
; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP10]])
-; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -2147483648, i32 [[IN]]
+; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -2147483648, i32 [[IN1]]
; IR-NEXT: [[TMP13:%.*]] = icmp sgt i32 [[TMP11]], [[TMP12]]
; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]]
; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4
@@ -845,12 +817,13 @@ define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr addrspace(1) %out, i
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.max.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile max ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile max ptr addrspace(1) [[GEP]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -871,15 +844,16 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64_offset(ptr addrspace(1) %ou
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.max.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile max ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile max ptr addrspace(1) [[GEP]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ]
; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP10]])
-; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -2147483648, i32 [[IN]]
+; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -2147483648, i32 [[IN1]]
; IR-NEXT: [[TMP13:%.*]] = icmp sgt i32 [[TMP11]], [[TMP12]]
; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]]
; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4
@@ -902,12 +876,13 @@ define amdgpu_kernel void @atomic_max_i32(ptr addrspace(1) %out, i32 %in) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.max.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile max ptr addrspace(1) [[OUT:%.*]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile max ptr addrspace(1) [[OUT:%.*]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -924,15 +899,16 @@ define amdgpu_kernel void @atomic_max_i32_ret(ptr addrspace(1) %out, ptr addrspa
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.max.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile max ptr addrspace(1) [[OUT:%.*]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile max ptr addrspace(1) [[OUT:%.*]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ]
; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP10]])
-; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -2147483648, i32 [[IN]]
+; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -2147483648, i32 [[IN1]]
; IR-NEXT: [[TMP13:%.*]] = icmp sgt i32 [[TMP11]], [[TMP12]]
; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]]
; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4
@@ -954,12 +930,13 @@ define amdgpu_kernel void @atomic_max_i32_addr64(ptr addrspace(1) %out, i32 %in,
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.max.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile max ptr addrspace(1) [[PTR]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile max ptr addrspace(1) [[PTR]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -978,15 +955,16 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64(ptr addrspace(1) %out, ptr
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.max.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile max ptr addrspace(1) [[PTR]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile max ptr addrspace(1) [[PTR]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ]
; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP10]])
-; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -2147483648, i32 [[IN]]
+; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -2147483648, i32 [[IN1]]
; IR-NEXT: [[TMP13:%.*]] = icmp sgt i32 [[TMP11]], [[TMP12]]
; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]]
; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4
@@ -1009,12 +987,13 @@ define amdgpu_kernel void @atomic_umax_i32_offset(ptr addrspace(1) %out, i32 %in
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.max.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile umax ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile umax ptr addrspace(1) [[GEP]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -1033,15 +1012,16 @@ define amdgpu_kernel void @atomic_umax_i32_ret_offset(ptr addrspace(1) %out, ptr
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.max.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile umax ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile umax ptr addrspace(1) [[GEP]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ]
; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP10]])
-; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 0, i32 [[IN]]
+; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 0, i32 [[IN1]]
; IR-NEXT: [[TMP13:%.*]] = icmp ugt i32 [[TMP11]], [[TMP12]]
; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]]
; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4
@@ -1065,12 +1045,13 @@ define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr addrspace(1) %out,
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.max.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile umax ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile umax ptr addrspace(1) [[GEP]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -1091,15 +1072,16 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64_offset(ptr addrspace(1) %o
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.max.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile umax ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile umax ptr addrspace(1) [[GEP]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ]
; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP10]])
-; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 0, i32 [[IN]]
+; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 0, i32 [[IN1]]
; IR-NEXT: [[TMP13:%.*]] = icmp ugt i32 [[TMP11]], [[TMP12]]
; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]]
; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4
@@ -1122,12 +1104,13 @@ define amdgpu_kernel void @atomic_umax_i32(ptr addrspace(1) %out, i32 %in) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.max.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile umax ptr addrspace(1) [[OUT:%.*]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile umax ptr addrspace(1) [[OUT:%.*]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -1144,15 +1127,16 @@ define amdgpu_kernel void @atomic_umax_i32_ret(ptr addrspace(1) %out, ptr addrsp
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.max.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile umax ptr addrspace(1) [[OUT:%.*]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile umax ptr addrspace(1) [[OUT:%.*]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ]
; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP10]])
-; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 0, i32 [[IN]]
+; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 0, i32 [[IN1]]
; IR-NEXT: [[TMP13:%.*]] = icmp ugt i32 [[TMP11]], [[TMP12]]
; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]]
; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4
@@ -1174,12 +1158,13 @@ define amdgpu_kernel void @atomic_umax_i32_addr64(ptr addrspace(1) %out, i32 %in
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.max.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile umax ptr addrspace(1) [[PTR]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile umax ptr addrspace(1) [[PTR]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -1198,15 +1183,16 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64(ptr addrspace(1) %out, ptr
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.max.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile umax ptr addrspace(1) [[PTR]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile umax ptr addrspace(1) [[PTR]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ]
; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP10]])
-; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 0, i32 [[IN]]
+; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 0, i32 [[IN1]]
; IR-NEXT: [[TMP13:%.*]] = icmp ugt i32 [[TMP11]], [[TMP12]]
; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]]
; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4
@@ -1229,12 +1215,13 @@ define amdgpu_kernel void @atomic_min_i32_offset(ptr addrspace(1) %out, i32 %in)
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.min.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile min ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile min ptr addrspace(1) [[GEP]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -1253,15 +1240,16 @@ define amdgpu_kernel void @atomic_min_i32_ret_offset(ptr addrspace(1) %out, ptr
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.min.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile min ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile min ptr addrspace(1) [[GEP]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ]
; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP10]])
-; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 2147483647, i32 [[IN]]
+; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 2147483647, i32 [[IN1]]
; IR-NEXT: [[TMP13:%.*]] = icmp slt i32 [[TMP11]], [[TMP12]]
; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]]
; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4
@@ -1285,12 +1273,13 @@ define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr addrspace(1) %out, i
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.min.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile min ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile min ptr addrspace(1) [[GEP]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -1311,15 +1300,16 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64_offset(ptr addrspace(1) %ou
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.min.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile min ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile min ptr addrspace(1) [[GEP]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ]
; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP10]])
-; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 2147483647, i32 [[IN]]
+; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 2147483647, i32 [[IN1]]
; IR-NEXT: [[TMP13:%.*]] = icmp slt i32 [[TMP11]], [[TMP12]]
; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]]
; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4
@@ -1342,12 +1332,13 @@ define amdgpu_kernel void @atomic_min_i32(ptr addrspace(1) %out, i32 %in) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.min.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile min ptr addrspace(1) [[OUT:%.*]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile min ptr addrspace(1) [[OUT:%.*]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -1364,15 +1355,16 @@ define amdgpu_kernel void @atomic_min_i32_ret(ptr addrspace(1) %out, ptr addrspa
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.min.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile min ptr addrspace(1) [[OUT:%.*]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile min ptr addrspace(1) [[OUT:%.*]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ]
; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP10]])
-; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 2147483647, i32 [[IN]]
+; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 2147483647, i32 [[IN1]]
; IR-NEXT: [[TMP13:%.*]] = icmp slt i32 [[TMP11]], [[TMP12]]
; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]]
; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4
@@ -1394,12 +1386,13 @@ define amdgpu_kernel void @atomic_min_i32_addr64(ptr addrspace(1) %out, i32 %in,
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.min.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile min ptr addrspace(1) [[PTR]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile min ptr addrspace(1) [[PTR]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -1418,15 +1411,16 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64(ptr addrspace(1) %out, ptr
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.min.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile min ptr addrspace(1) [[PTR]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile min ptr addrspace(1) [[PTR]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ]
; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP10]])
-; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 2147483647, i32 [[IN]]
+; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 2147483647, i32 [[IN1]]
; IR-NEXT: [[TMP13:%.*]] = icmp slt i32 [[TMP11]], [[TMP12]]
; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]]
; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd-wrong-subtarget.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd-wrong-subtarget.ll
index a1da18969552a..31d439f21c297 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd-wrong-subtarget.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd-wrong-subtarget.ll
@@ -5,31 +5,32 @@ define amdgpu_kernel void @global_atomic_fadd_ret_f32_wrong_subtarget(ptr addrsp
; GCN-LABEL: global_atomic_fadd_ret_f32_wrong_subtarget:
; GCN: ; %bb.0:
; GCN-NEXT: s_mov_b64 s[0:1], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
+; GCN-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GCN-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GCN-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT: v_mul_f32_e32 v1, 4.0, v1
+; GCN-NEXT: v_readfirstlane_b32 s8, v1
; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GCN-NEXT: s_cbranch_execz .LBB0_4
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x0
-; GCN-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GCN-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GCN-NEXT: s_mov_b64 s[6:7], 0
-; GCN-NEXT: v_mul_f32_e32 v2, 4.0, v1
+; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_load_dword s8, s[4:5], 0x0
-; GCN-NEXT: v_mov_b32_e32 v3, 0
+; GCN-NEXT: s_load_dword s0, s[4:5], 0x0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v1, s8
+; GCN-NEXT: v_mov_b32_e32 v1, s0
; GCN-NEXT: .LBB0_2: ; %atomicrmw.start
; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN-NEXT: v_mov_b32_e32 v5, v1
-; GCN-NEXT: v_add_f32_e32 v4, v5, v2
-; GCN-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[4:5] glc
+; GCN-NEXT: v_mov_b32_e32 v4, v1
+; GCN-NEXT: v_add_f32_e32 v3, s8, v4
+; GCN-NEXT: global_atomic_cmpswap v1, v2, v[3:4], s[4:5] glc
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_wbinvl1
-; GCN-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v5
+; GCN-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v4
; GCN-NEXT: s_or_b64 s[6:7], s[0:1], s[6:7]
; GCN-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GCN-NEXT: s_cbranch_execnz .LBB0_2
@@ -53,17 +54,19 @@ define amdgpu_kernel void @global_atomic_fadd_noret_f32_wrong_subtarget(ptr addr
; GCN-LABEL: global_atomic_fadd_noret_f32_wrong_subtarget:
; GCN: ; %bb.0:
; GCN-NEXT: s_mov_b64 s[0:1], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GCN-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GCN-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GCN-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GCN-NEXT: v_readfirstlane_b32 s0, v0
; GCN-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GCN-NEXT: s_cbranch_execz .LBB1_2
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
-; GCN-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GCN-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GCN-NEXT: v_mov_b32_e32 v0, 0
-; GCN-NEXT: v_mul_f32_e32 v1, 4.0, v1
+; GCN-NEXT: v_mov_b32_e32 v1, s0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GCN-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomics-fp-wrong-subtarget.ll b/llvm/test/CodeGen/AMDGPU/global-atomics-fp-wrong-subtarget.ll
index 2f5d579d34a5a..af8d2a47075de 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomics-fp-wrong-subtarget.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomics-fp-wrong-subtarget.ll
@@ -12,17 +12,19 @@ define amdgpu_kernel void @global_atomic_fadd_noret_f32_wrong_subtarget(ptr addr
; GCN-LABEL: global_atomic_fadd_noret_f32_wrong_subtarget:
; GCN: ; %bb.0:
; GCN-NEXT: s_mov_b64 s[0:1], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GCN-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GCN-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GCN-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GCN-NEXT: v_readfirstlane_b32 s0, v0
; GCN-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GCN-NEXT: s_cbranch_execz .LBB0_2
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GCN-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GCN-NEXT: v_mov_b32_e32 v0, 0
-; GCN-NEXT: v_mul_f32_e32 v1, 4.0, v1
+; GCN-NEXT: v_mov_b32_e32 v1, s0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GCN-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll b/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll
index a09643b859232..376653c5b3d0c 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll
@@ -7,36 +7,61 @@
; strategies are valid for only divergent values. This optimization is valid for divergent addresses. Test also covers different scopes.
define amdgpu_ps float @global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, float inreg %val) #0 {
-; IR-LABEL: @global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe(
-; IR-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
-; IR-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]]
-; IR: 2:
-; IR-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
-; IR-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
-; IR-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
-; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
-; IR-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]])
-; IR-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-NEXT: [[TMP11:%.*]] = uitofp i32 [[TMP10]] to float
-; IR-NEXT: [[TMP12:%.*]] = fmul float [[VAL:%.*]], [[TMP11]]
-; IR-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR: 14:
-; IR-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] syncscope("agent") monotonic, align 4
-; IR-NEXT: br label [[TMP16]]
-; IR: 16:
-; IR-NEXT: [[TMP17:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
-; IR-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP17]])
-; IR-NEXT: [[TMP19:%.*]] = uitofp i32 [[TMP8]] to float
-; IR-NEXT: [[TMP20:%.*]] = fmul float [[VAL]], [[TMP19]]
-; IR-NEXT: [[TMP21:%.*]] = fadd float [[TMP18]], [[TMP20]]
-; IR-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], float [[TMP18]], float [[TMP21]]
-; IR-NEXT: br label [[TMP23]]
-; IR: 23:
-; IR-NEXT: [[TMP24:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ]
-; IR-NEXT: ret float [[TMP24]]
+; IR-ITERATIVE-LABEL: @global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe(
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
+; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP20:%.*]]
+; IR-ITERATIVE: 2:
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 1)
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP9]] syncscope("agent") monotonic, align 4
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP12]], [[TMP11]] ]
+; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP14]])
+; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = uitofp i32 [[TMP8]] to float
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = fmul float [[VAL]], [[TMP16]]
+; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = fadd float [[TMP15]], [[TMP17]]
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = select i1 [[TMP10]], float [[TMP15]], float [[TMP18]]
+; IR-ITERATIVE-NEXT: br label [[TMP20]]
+; IR-ITERATIVE: 20:
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP19]], [[TMP13]] ]
+; IR-ITERATIVE-NEXT: ret float [[TMP21]]
+;
+; IR-DPP-LABEL: @global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe(
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
+; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP20:%.*]]
+; IR-DPP: 2:
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
+; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 2)
+; IR-DPP-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-DPP-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]
+; IR-DPP: 11:
+; IR-DPP-NEXT: [[TMP12:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP9]] syncscope("agent") monotonic, align 4
+; IR-DPP-NEXT: br label [[TMP13]]
+; IR-DPP: 13:
+; IR-DPP-NEXT: [[TMP14:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP12]], [[TMP11]] ]
+; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP14]])
+; IR-DPP-NEXT: [[TMP16:%.*]] = uitofp i32 [[TMP8]] to float
+; IR-DPP-NEXT: [[TMP17:%.*]] = fmul float [[VAL]], [[TMP16]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = fadd float [[TMP15]], [[TMP17]]
+; IR-DPP-NEXT: [[TMP19:%.*]] = select i1 [[TMP10]], float [[TMP15]], float [[TMP18]]
+; IR-DPP-NEXT: br label [[TMP20]]
+; IR-DPP: 20:
+; IR-DPP-NEXT: [[TMP21:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP19]], [[TMP13]] ]
+; IR-DPP-NEXT: ret float [[TMP21]]
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, float %val syncscope("agent") monotonic, align 4
ret float %result
@@ -133,64 +158,58 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_div_value_scope_agent_sco
define amdgpu_ps float @global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, float inreg %val) #1 {
; IR-ITERATIVE-LABEL: @global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7:[0-9]+]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2:[0-9]+]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 1) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-ITERATIVE: 14:
+; IR-ITERATIVE: 11:
; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] syncscope("one-as") monotonic, align 4
; IR-ITERATIVE-NEXT: br label [[TMP16]]
-; IR-ITERATIVE: 16:
+; IR-ITERATIVE: 13:
; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP17]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP18]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP17]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP18]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], float [[TMP18]], float [[TMP21]]
; IR-ITERATIVE-NEXT: br label [[TMP23]]
-; IR-ITERATIVE: 23:
+; IR-ITERATIVE: 20:
; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ]
; IR-ITERATIVE-NEXT: ret float [[TMP24]]
;
; IR-DPP-LABEL: @global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8:[0-9]+]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2:[0-9]+]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-DPP: 14:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] syncscope("one-as") monotonic, align 4
; IR-DPP-NEXT: br label [[TMP16]]
-; IR-DPP: 16:
+; IR-DPP: 13:
; IR-DPP-NEXT: [[TMP17:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP17]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP18]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP17]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP18]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-DPP-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], float [[TMP18]], float [[TMP21]]
; IR-DPP-NEXT: br label [[TMP23]]
-; IR-DPP: 23:
+; IR-DPP: 20:
; IR-DPP-NEXT: [[TMP24:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ]
; IR-DPP-NEXT: ret float [[TMP24]]
;
@@ -200,24 +219,24 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_uni_value_one_as_scope_un
define amdgpu_ps float @global_atomic_fadd_uni_address_div_value_one_as_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, float %val) #1 {
; IR-ITERATIVE-LABEL: @global_atomic_fadd_uni_address_div_value_one_as_scope_unsafe_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
; IR-ITERATIVE: 10:
; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP23:%.*]] syncscope("one-as") monotonic, align 4
; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
; IR-ITERATIVE: 12:
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = phi float [ poison, [[COMPUTEEND:%.*]] ], [ [[TMP11]], [[TMP10:%.*]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP14]], float [[TMP22:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP14]], float [[TMP22:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = select i1 [[TMP28:%.*]], float [[TMP14]], float [[TMP15]]
; IR-ITERATIVE-NEXT: br label [[TMP17]]
; IR-ITERATIVE: 17:
@@ -227,11 +246,11 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_div_value_one_as_scope_un
; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ -0.000000e+00, [[TMP2]] ], [ [[TMP23]], [[COMPUTELOOP]] ]
; IR-ITERATIVE-NEXT: [[OLDVALUEPHI:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP22]], [[COMPUTELOOP]] ]
; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP26:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = trunc i64 [[TMP19]] to i32
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP22]] = call float @llvm.amdgcn.writelane.f32(float [[ACCUMULATOR]], i32 [[TMP20]], float [[OLDVALUEPHI]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP23]] = call float @llvm.experimental.constrained.fadd.f32(float [[ACCUMULATOR]], float [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP22]] = call float @llvm.amdgcn.writelane.f32(float [[ACCUMULATOR]], i32 [[TMP20]], float [[OLDVALUEPHI]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP23]] = call float @llvm.experimental.constrained.fadd.f32(float [[ACCUMULATOR]], float [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = shl i64 1, [[TMP19]]
; IR-ITERATIVE-NEXT: [[TMP25:%.*]] = xor i64 [[TMP24]], -1
; IR-ITERATIVE-NEXT: [[TMP26]] = and i64 [[ACTIVEBITS]], [[TMP25]]
@@ -242,31 +261,31 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_div_value_one_as_scope_un
; IR-ITERATIVE-NEXT: br i1 [[TMP28]], label [[TMP10]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fadd_uni_address_div_value_one_as_scope_unsafe_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP34:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[VAL:%.*]], float -0.000000e+00) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP9]], float [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP13:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP11]], float [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP13]], float [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP15]], float [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP17]], float [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP19]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP21]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP24:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP23]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[VAL:%.*]], float -0.000000e+00) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP9]], float [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP13:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP11]], float [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP13]], float [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP15]], float [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP17]], float [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP19]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP22:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP21]], i32 63) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP24:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP23]]) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP25:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP28:%.*]]
; IR-DPP: 26:
@@ -274,9 +293,9 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_div_value_one_as_scope_un
; IR-DPP-NEXT: br label [[TMP28]]
; IR-DPP: 28:
; IR-DPP-NEXT: [[TMP29:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP27]], [[TMP26]] ]
-; IR-DPP-NEXT: [[TMP30:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP29]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP31:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP32:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP30]], float [[TMP31]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP30:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP29]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP31:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP32:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP30]], float [[TMP31]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-DPP-NEXT: [[TMP33:%.*]] = select i1 [[TMP25]], float [[TMP30]], float [[TMP32]]
; IR-DPP-NEXT: br label [[TMP34]]
; IR-DPP: 34:
@@ -289,64 +308,58 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_div_value_one_as_scope_un
define amdgpu_ps float @global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp(ptr addrspace(1) inreg %ptr, float inreg %val) #2 {
; IR-ITERATIVE-LABEL: @global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 1) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-ITERATIVE: 14:
+; IR-ITERATIVE: 11:
; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] syncscope("agent") monotonic, align 4
; IR-ITERATIVE-NEXT: br label [[TMP16]]
-; IR-ITERATIVE: 16:
+; IR-ITERATIVE: 13:
; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP17]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP18]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP17]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP18]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], float [[TMP18]], float [[TMP21]]
; IR-ITERATIVE-NEXT: br label [[TMP23]]
-; IR-ITERATIVE: 23:
+; IR-ITERATIVE: 20:
; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ]
; IR-ITERATIVE-NEXT: ret float [[TMP24]]
;
; IR-DPP-LABEL: @global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-DPP: 14:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] syncscope("agent") monotonic, align 4
; IR-DPP-NEXT: br label [[TMP16]]
-; IR-DPP: 16:
+; IR-DPP: 13:
; IR-DPP-NEXT: [[TMP17:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP17]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP18]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP17]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP18]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-DPP-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], float [[TMP18]], float [[TMP21]]
; IR-DPP-NEXT: br label [[TMP23]]
-; IR-DPP: 23:
+; IR-DPP: 20:
; IR-DPP-NEXT: [[TMP24:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ]
; IR-DPP-NEXT: ret float [[TMP24]]
;
@@ -356,24 +369,24 @@ define amdgpu_ps float @global_atomic_fsub_uni_address_uni_value_agent_scope_str
define amdgpu_ps float @global_atomic_fsub_uni_address_div_value_agent_scope_strictfp(ptr addrspace(1) inreg %ptr, float %val) #2 {
; IR-ITERATIVE-LABEL: @global_atomic_fsub_uni_address_div_value_agent_scope_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
; IR-ITERATIVE: 10:
; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], float [[TMP23:%.*]] syncscope("agent") monotonic, align 4
; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
; IR-ITERATIVE: 12:
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = phi float [ poison, [[COMPUTEEND:%.*]] ], [ [[TMP11]], [[TMP10:%.*]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fsub.f32(float [[TMP14]], float [[TMP22:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fsub.f32(float [[TMP14]], float [[TMP22:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = select i1 [[TMP28:%.*]], float [[TMP14]], float [[TMP15]]
; IR-ITERATIVE-NEXT: br label [[TMP17]]
; IR-ITERATIVE: 17:
@@ -383,11 +396,11 @@ define amdgpu_ps float @global_atomic_fsub_uni_address_div_value_agent_scope_str
; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ -0.000000e+00, [[TMP2]] ], [ [[TMP23]], [[COMPUTELOOP]] ]
; IR-ITERATIVE-NEXT: [[OLDVALUEPHI:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP22]], [[COMPUTELOOP]] ]
; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP26:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = trunc i64 [[TMP19]] to i32
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP22]] = call float @llvm.amdgcn.writelane.f32(float [[ACCUMULATOR]], i32 [[TMP20]], float [[OLDVALUEPHI]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP23]] = call float @llvm.experimental.constrained.fadd.f32(float [[ACCUMULATOR]], float [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP22]] = call float @llvm.amdgcn.writelane.f32(float [[ACCUMULATOR]], i32 [[TMP20]], float [[OLDVALUEPHI]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP23]] = call float @llvm.experimental.constrained.fadd.f32(float [[ACCUMULATOR]], float [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = shl i64 1, [[TMP19]]
; IR-ITERATIVE-NEXT: [[TMP25:%.*]] = xor i64 [[TMP24]], -1
; IR-ITERATIVE-NEXT: [[TMP26]] = and i64 [[ACTIVEBITS]], [[TMP25]]
@@ -398,31 +411,31 @@ define amdgpu_ps float @global_atomic_fsub_uni_address_div_value_agent_scope_str
; IR-ITERATIVE-NEXT: br i1 [[TMP28]], label [[TMP10]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fsub_uni_address_div_value_agent_scope_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP34:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[VAL:%.*]], float -0.000000e+00) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP9]], float [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP13:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP11]], float [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP13]], float [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP15]], float [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP17]], float [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP19]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP21]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP24:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP23]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[VAL:%.*]], float -0.000000e+00) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP9]], float [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP13:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP11]], float [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP13]], float [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP15]], float [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP17]], float [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP19]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP22:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP21]], i32 63) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP24:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP23]]) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP25:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP28:%.*]]
; IR-DPP: 26:
@@ -430,9 +443,9 @@ define amdgpu_ps float @global_atomic_fsub_uni_address_div_value_agent_scope_str
; IR-DPP-NEXT: br label [[TMP28]]
; IR-DPP: 28:
; IR-DPP-NEXT: [[TMP29:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP27]], [[TMP26]] ]
-; IR-DPP-NEXT: [[TMP30:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP29]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP31:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP32:%.*]] = call float @llvm.experimental.constrained.fsub.f32(float [[TMP30]], float [[TMP31]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP30:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP29]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP31:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP32:%.*]] = call float @llvm.experimental.constrained.fsub.f32(float [[TMP30]], float [[TMP31]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-DPP-NEXT: [[TMP33:%.*]] = select i1 [[TMP25]], float [[TMP30]], float [[TMP32]]
; IR-DPP-NEXT: br label [[TMP34]]
; IR-DPP: 34:
@@ -444,32 +457,61 @@ define amdgpu_ps float @global_atomic_fsub_uni_address_div_value_agent_scope_str
}
define amdgpu_ps float @global_atomic_fmin_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, float inreg %val) #0 {
-; IR-LABEL: @global_atomic_fmin_uni_address_uni_value_agent_scope_unsafe(
-; IR-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
-; IR-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP19:%.*]]
-; IR: 2:
-; IR-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
-; IR-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
-; IR-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
-; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
-; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
-; IR-NEXT: [[TMP11:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], float [[VAL:%.*]] syncscope("agent") monotonic, align 4
-; IR-NEXT: br label [[TMP12]]
-; IR: 12:
-; IR-NEXT: [[TMP13:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP11]], [[TMP10]] ]
-; IR-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]])
-; IR-NEXT: [[TMP15:%.*]] = uitofp i32 [[TMP8]] to float
-; IR-NEXT: [[TMP16:%.*]] = select i1 [[TMP9]], float +qnan, float [[VAL]]
-; IR-NEXT: [[TMP17:%.*]] = call float @llvm.minnum.f32(float [[TMP14]], float [[TMP16]])
-; IR-NEXT: [[TMP18:%.*]] = select i1 [[TMP9]], float [[TMP14]], float [[TMP17]]
-; IR-NEXT: br label [[TMP19]]
-; IR: 19:
-; IR-NEXT: [[TMP20:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP18]], [[TMP12]] ]
-; IR-NEXT: ret float [[TMP20]]
+; IR-ITERATIVE-LABEL: @global_atomic_fmin_uni_address_uni_value_agent_scope_unsafe(
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
+; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP20:%.*]]
+; IR-ITERATIVE: 2:
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.wave.reduce.fmin.f32(float [[VAL:%.*]], i32 1)
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], float [[TMP9]] syncscope("agent") monotonic, align 4
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP12]], [[TMP11]] ]
+; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP14]])
+; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = uitofp i32 [[TMP8]] to float
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = select i1 [[TMP10]], float +qnan, float [[VAL]]
+; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call float @llvm.minnum.f32(float [[TMP15]], float [[TMP17]])
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = select i1 [[TMP10]], float [[TMP15]], float [[TMP18]]
+; IR-ITERATIVE-NEXT: br label [[TMP20]]
+; IR-ITERATIVE: 20:
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP19]], [[TMP13]] ]
+; IR-ITERATIVE-NEXT: ret float [[TMP21]]
+;
+; IR-DPP-LABEL: @global_atomic_fmin_uni_address_uni_value_agent_scope_unsafe(
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
+; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP20:%.*]]
+; IR-DPP: 2:
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
+; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.wave.reduce.fmin.f32(float [[VAL:%.*]], i32 2)
+; IR-DPP-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-DPP-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]
+; IR-DPP: 11:
+; IR-DPP-NEXT: [[TMP12:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], float [[TMP9]] syncscope("agent") monotonic, align 4
+; IR-DPP-NEXT: br label [[TMP13]]
+; IR-DPP: 13:
+; IR-DPP-NEXT: [[TMP14:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP12]], [[TMP11]] ]
+; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP14]])
+; IR-DPP-NEXT: [[TMP16:%.*]] = uitofp i32 [[TMP8]] to float
+; IR-DPP-NEXT: [[TMP17:%.*]] = select i1 [[TMP10]], float +qnan, float [[VAL]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.minnum.f32(float [[TMP15]], float [[TMP17]])
+; IR-DPP-NEXT: [[TMP19:%.*]] = select i1 [[TMP10]], float [[TMP15]], float [[TMP18]]
+; IR-DPP-NEXT: br label [[TMP20]]
+; IR-DPP: 20:
+; IR-DPP-NEXT: [[TMP21:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP19]], [[TMP13]] ]
+; IR-DPP-NEXT: ret float [[TMP21]]
;
%result = atomicrmw fmin ptr addrspace(1) %ptr, float %val syncscope("agent") monotonic
ret float %result
@@ -566,56 +608,58 @@ define amdgpu_ps float @global_atomic_fmin_uni_address_div_value_agent_scope_uns
define amdgpu_ps float @global_atomic_fmax_uni_address_uni_value_agent_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, float inreg %val) #1{
; IR-ITERATIVE-LABEL: @global_atomic_fmax_uni_address_uni_value_agent_scope_unsafe_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP19:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.wave.reduce.fmax.f32(float [[VAL1:%.*]], i32 1) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR-ITERATIVE: 10:
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], float [[VAL:%.*]] syncscope("agent") monotonic, align 4
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], float [[VAL]] syncscope("agent") monotonic, align 4
; IR-ITERATIVE-NEXT: br label [[TMP12]]
-; IR-ITERATIVE: 12:
+; IR-ITERATIVE: 13:
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP11]], [[TMP10]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = select i1 [[TMP9]], float +qnan, float [[VAL]]
-; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP14]], float [[TMP16]], metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = select i1 [[TMP9]], float +qnan, float [[VAL1]]
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP14]], float [[TMP21]], metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = select i1 [[TMP9]], float [[TMP14]], float [[TMP17]]
; IR-ITERATIVE-NEXT: br label [[TMP19]]
-; IR-ITERATIVE: 19:
+; IR-ITERATIVE: 20:
; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP18]], [[TMP12]] ]
; IR-ITERATIVE-NEXT: ret float [[TMP20]]
;
; IR-DPP-LABEL: @global_atomic_fmax_uni_address_uni_value_agent_scope_unsafe_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP19:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.wave.reduce.fmax.f32(float [[VAL1:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR-DPP: 10:
-; IR-DPP-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], float [[VAL:%.*]] syncscope("agent") monotonic, align 4
+; IR-DPP: 11:
+; IR-DPP-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], float [[VAL]] syncscope("agent") monotonic, align 4
; IR-DPP-NEXT: br label [[TMP12]]
-; IR-DPP: 12:
+; IR-DPP: 13:
; IR-DPP-NEXT: [[TMP13:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP11]], [[TMP10]] ]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = select i1 [[TMP9]], float +qnan, float [[VAL]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP14]], float [[TMP16]], metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP21:%.*]] = select i1 [[TMP9]], float +qnan, float [[VAL1]]
+; IR-DPP-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP14]], float [[TMP21]], metadata !"fpexcept.strict") #[[ATTR2]]
; IR-DPP-NEXT: [[TMP18:%.*]] = select i1 [[TMP9]], float [[TMP14]], float [[TMP17]]
; IR-DPP-NEXT: br label [[TMP19]]
-; IR-DPP: 19:
+; IR-DPP: 20:
; IR-DPP-NEXT: [[TMP20:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP18]], [[TMP12]] ]
; IR-DPP-NEXT: ret float [[TMP20]]
;
@@ -625,24 +669,24 @@ define amdgpu_ps float @global_atomic_fmax_uni_address_uni_value_agent_scope_uns
define amdgpu_ps float @global_atomic_fmax_uni_address_div_value_agent_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, float %val) #1{
; IR-ITERATIVE-LABEL: @global_atomic_fmax_uni_address_div_value_agent_scope_unsafe_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
; IR-ITERATIVE: 10:
; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], float [[TMP23:%.*]] syncscope("agent") monotonic, align 4
; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
; IR-ITERATIVE: 12:
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = phi float [ poison, [[COMPUTEEND:%.*]] ], [ [[TMP11]], [[TMP10:%.*]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP14]], float [[TMP22:%.*]], metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP14]], float [[TMP22:%.*]], metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = select i1 [[TMP28:%.*]], float [[TMP14]], float [[TMP15]]
; IR-ITERATIVE-NEXT: br label [[TMP17]]
; IR-ITERATIVE: 17:
@@ -652,11 +696,11 @@ define amdgpu_ps float @global_atomic_fmax_uni_address_div_value_agent_scope_uns
; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ +qnan, [[TMP2]] ], [ [[TMP23]], [[COMPUTELOOP]] ]
; IR-ITERATIVE-NEXT: [[OLDVALUEPHI:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP22]], [[COMPUTELOOP]] ]
; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP26:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = trunc i64 [[TMP19]] to i32
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP22]] = call float @llvm.amdgcn.writelane.f32(float [[ACCUMULATOR]], i32 [[TMP20]], float [[OLDVALUEPHI]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP23]] = call float @llvm.experimental.constrained.maxnum.f32(float [[ACCUMULATOR]], float [[TMP21]], metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP22]] = call float @llvm.amdgcn.writelane.f32(float [[ACCUMULATOR]], i32 [[TMP20]], float [[OLDVALUEPHI]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP23]] = call float @llvm.experimental.constrained.maxnum.f32(float [[ACCUMULATOR]], float [[TMP21]], metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = shl i64 1, [[TMP19]]
; IR-ITERATIVE-NEXT: [[TMP25:%.*]] = xor i64 [[TMP24]], -1
; IR-ITERATIVE-NEXT: [[TMP26]] = and i64 [[ACTIVEBITS]], [[TMP25]]
@@ -667,31 +711,31 @@ define amdgpu_ps float @global_atomic_fmax_uni_address_div_value_agent_scope_uns
; IR-ITERATIVE-NEXT: br i1 [[TMP28]], label [[TMP10]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fmax_uni_address_div_value_agent_scope_unsafe_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP34:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[VAL:%.*]], float +qnan) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP9]], float [[TMP10]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP13:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP11]], float [[TMP12]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP13]], float [[TMP14]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP15]], float [[TMP16]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP17]], float [[TMP18]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP19]], float [[TMP20]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP21]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP24:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP23]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[VAL:%.*]], float +qnan) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP9]], float [[TMP10]], metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP13:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP11]], float [[TMP12]], metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP13]], float [[TMP14]], metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP15]], float [[TMP16]], metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP17]], float [[TMP18]], metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP19]], float [[TMP20]], metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP22:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP21]], i32 63) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP24:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP23]]) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP25:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP28:%.*]]
; IR-DPP: 26:
@@ -699,9 +743,9 @@ define amdgpu_ps float @global_atomic_fmax_uni_address_div_value_agent_scope_uns
; IR-DPP-NEXT: br label [[TMP28]]
; IR-DPP: 28:
; IR-DPP-NEXT: [[TMP29:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP27]], [[TMP26]] ]
-; IR-DPP-NEXT: [[TMP30:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP29]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP31:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP32:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP30]], float [[TMP31]], metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP30:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP29]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP31:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP32:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP30]], float [[TMP31]], metadata !"fpexcept.strict") #[[ATTR2]]
; IR-DPP-NEXT: [[TMP33:%.*]] = select i1 [[TMP25]], float [[TMP30]], float [[TMP32]]
; IR-DPP-NEXT: br label [[TMP34]]
; IR-DPP: 34:
@@ -714,64 +758,58 @@ define amdgpu_ps float @global_atomic_fmax_uni_address_div_value_agent_scope_uns
define amdgpu_ps float @global_atomic_fadd_uni_address_uni_value_system_scope_strictfp(ptr addrspace(1) inreg %ptr, float inreg %val) #2 {
; IR-ITERATIVE-LABEL: @global_atomic_fadd_uni_address_uni_value_system_scope_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 1) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-ITERATIVE: 14:
+; IR-ITERATIVE: 11:
; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] monotonic, align 4
; IR-ITERATIVE-NEXT: br label [[TMP16]]
-; IR-ITERATIVE: 16:
+; IR-ITERATIVE: 13:
; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP17]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP18]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP17]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP18]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], float [[TMP18]], float [[TMP21]]
; IR-ITERATIVE-NEXT: br label [[TMP23]]
-; IR-ITERATIVE: 23:
+; IR-ITERATIVE: 20:
; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ]
; IR-ITERATIVE-NEXT: ret float [[TMP24]]
;
; IR-DPP-LABEL: @global_atomic_fadd_uni_address_uni_value_system_scope_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-DPP: 14:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] monotonic, align 4
; IR-DPP-NEXT: br label [[TMP16]]
-; IR-DPP: 16:
+; IR-DPP: 13:
; IR-DPP-NEXT: [[TMP17:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP17]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP18]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP17]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP18]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-DPP-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], float [[TMP18]], float [[TMP21]]
; IR-DPP-NEXT: br label [[TMP23]]
-; IR-DPP: 23:
+; IR-DPP: 20:
; IR-DPP-NEXT: [[TMP24:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ]
; IR-DPP-NEXT: ret float [[TMP24]]
;
@@ -781,24 +819,24 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_uni_value_system_scope_st
define amdgpu_ps float @global_atomic_fadd_uni_address_div_value_system_scope_strictfp(ptr addrspace(1) inreg %ptr, float %val) #2 {
; IR-ITERATIVE-LABEL: @global_atomic_fadd_uni_address_div_value_system_scope_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
; IR-ITERATIVE: 10:
; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP23:%.*]] monotonic, align 4
; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
; IR-ITERATIVE: 12:
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = phi float [ poison, [[COMPUTEEND:%.*]] ], [ [[TMP11]], [[TMP10:%.*]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP14]], float [[TMP22:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP14]], float [[TMP22:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = select i1 [[TMP28:%.*]], float [[TMP14]], float [[TMP15]]
; IR-ITERATIVE-NEXT: br label [[TMP17]]
; IR-ITERATIVE: 17:
@@ -808,11 +846,11 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_div_value_system_scope_st
; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ -0.000000e+00, [[TMP2]] ], [ [[TMP23]], [[COMPUTELOOP]] ]
; IR-ITERATIVE-NEXT: [[OLDVALUEPHI:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP22]], [[COMPUTELOOP]] ]
; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP26:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = trunc i64 [[TMP19]] to i32
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP22]] = call float @llvm.amdgcn.writelane.f32(float [[ACCUMULATOR]], i32 [[TMP20]], float [[OLDVALUEPHI]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP23]] = call float @llvm.experimental.constrained.fadd.f32(float [[ACCUMULATOR]], float [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP22]] = call float @llvm.amdgcn.writelane.f32(float [[ACCUMULATOR]], i32 [[TMP20]], float [[OLDVALUEPHI]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP23]] = call float @llvm.experimental.constrained.fadd.f32(float [[ACCUMULATOR]], float [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = shl i64 1, [[TMP19]]
; IR-ITERATIVE-NEXT: [[TMP25:%.*]] = xor i64 [[TMP24]], -1
; IR-ITERATIVE-NEXT: [[TMP26]] = and i64 [[ACTIVEBITS]], [[TMP25]]
@@ -823,31 +861,31 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_div_value_system_scope_st
; IR-ITERATIVE-NEXT: br i1 [[TMP28]], label [[TMP10]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fadd_uni_address_div_value_system_scope_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP34:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[VAL:%.*]], float -0.000000e+00) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP9]], float [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP13:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP11]], float [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP13]], float [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP15]], float [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP17]], float [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP19]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP21]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP24:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP23]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[VAL:%.*]], float -0.000000e+00) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP9]], float [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP13:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP11]], float [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP13]], float [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP15]], float [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP17]], float [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP19]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP22:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP21]], i32 63) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP24:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP23]]) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP25:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP28:%.*]]
; IR-DPP: 26:
@@ -855,9 +893,9 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_div_value_system_scope_st
; IR-DPP-NEXT: br label [[TMP28]]
; IR-DPP: 28:
; IR-DPP-NEXT: [[TMP29:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP27]], [[TMP26]] ]
-; IR-DPP-NEXT: [[TMP30:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP29]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP31:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP32:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP30]], float [[TMP31]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP30:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP29]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP31:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP32:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP30]], float [[TMP31]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-DPP-NEXT: [[TMP33:%.*]] = select i1 [[TMP25]], float [[TMP30]], float [[TMP32]]
; IR-DPP-NEXT: br label [[TMP34]]
; IR-DPP: 34:
@@ -977,36 +1015,61 @@ define amdgpu_ps float @global_atomic_fadd_div_address_div_value_system_scope_st
}
define amdgpu_ps double @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double inreg %val) #0 {
-; IR-LABEL: @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe(
-; IR-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
-; IR-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]]
-; IR: 2:
-; IR-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
-; IR-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
-; IR-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
-; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
-; IR-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]])
-; IR-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-NEXT: [[TMP11:%.*]] = uitofp i32 [[TMP10]] to double
-; IR-NEXT: [[TMP12:%.*]] = fmul double [[VAL:%.*]], [[TMP11]]
-; IR-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR: 14:
-; IR-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 4
-; IR-NEXT: br label [[TMP16]]
-; IR: 16:
-; IR-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
-; IR-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP17]])
-; IR-NEXT: [[TMP19:%.*]] = uitofp i32 [[TMP8]] to double
-; IR-NEXT: [[TMP20:%.*]] = fmul double [[VAL]], [[TMP19]]
-; IR-NEXT: [[TMP21:%.*]] = fadd double [[TMP18]], [[TMP20]]
-; IR-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], double [[TMP18]], double [[TMP21]]
-; IR-NEXT: br label [[TMP23]]
-; IR: 23:
-; IR-NEXT: [[TMP24:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ]
-; IR-NEXT: ret double [[TMP24]]
+; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe(
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
+; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP20:%.*]]
+; IR-ITERATIVE: 2:
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 1)
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP9]] syncscope("agent") monotonic, align 4
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP12]], [[TMP11]] ]
+; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP14]])
+; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = uitofp i32 [[TMP8]] to double
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = fmul double [[VAL]], [[TMP16]]
+; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = fadd double [[TMP15]], [[TMP17]]
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = select i1 [[TMP10]], double [[TMP15]], double [[TMP18]]
+; IR-ITERATIVE-NEXT: br label [[TMP20]]
+; IR-ITERATIVE: 20:
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP19]], [[TMP13]] ]
+; IR-ITERATIVE-NEXT: ret double [[TMP21]]
+;
+; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe(
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
+; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP20:%.*]]
+; IR-DPP: 2:
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
+; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 2)
+; IR-DPP-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-DPP-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]
+; IR-DPP: 11:
+; IR-DPP-NEXT: [[TMP12:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP9]] syncscope("agent") monotonic, align 4
+; IR-DPP-NEXT: br label [[TMP13]]
+; IR-DPP: 13:
+; IR-DPP-NEXT: [[TMP14:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP12]], [[TMP11]] ]
+; IR-DPP-NEXT: [[TMP15:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP14]])
+; IR-DPP-NEXT: [[TMP16:%.*]] = uitofp i32 [[TMP8]] to double
+; IR-DPP-NEXT: [[TMP17:%.*]] = fmul double [[VAL]], [[TMP16]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = fadd double [[TMP15]], [[TMP17]]
+; IR-DPP-NEXT: [[TMP19:%.*]] = select i1 [[TMP10]], double [[TMP15]], double [[TMP18]]
+; IR-DPP-NEXT: br label [[TMP20]]
+; IR-DPP: 20:
+; IR-DPP-NEXT: [[TMP21:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP19]], [[TMP13]] ]
+; IR-DPP-NEXT: ret double [[TMP21]]
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic, align 4
ret double %result
@@ -1103,64 +1166,58 @@ define amdgpu_ps double @global_atomic_fadd_double_uni_address_div_value_scope_a
define amdgpu_ps double @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #1 {
; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 1) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-ITERATIVE: 14:
+; IR-ITERATIVE: 11:
; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("one-as") monotonic, align 8
; IR-ITERATIVE-NEXT: br label [[TMP16]]
-; IR-ITERATIVE: 16:
+; IR-ITERATIVE: 13:
; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP17]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP18]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP17]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP18]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], double [[TMP18]], double [[TMP21]]
; IR-ITERATIVE-NEXT: br label [[TMP23]]
-; IR-ITERATIVE: 23:
+; IR-ITERATIVE: 20:
; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ]
; IR-ITERATIVE-NEXT: ret double [[TMP24]]
;
; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-DPP: 14:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("one-as") monotonic, align 8
; IR-DPP-NEXT: br label [[TMP16]]
-; IR-DPP: 16:
+; IR-DPP: 13:
; IR-DPP-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP17]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP18]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP17]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP18]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-DPP-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], double [[TMP18]], double [[TMP21]]
; IR-DPP-NEXT: br label [[TMP23]]
-; IR-DPP: 23:
+; IR-DPP: 20:
; IR-DPP-NEXT: [[TMP24:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ]
; IR-DPP-NEXT: ret double [[TMP24]]
;
@@ -1170,24 +1227,24 @@ define amdgpu_ps double @global_atomic_fadd_double_uni_address_uni_value_one_as_
define amdgpu_ps double @global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, double %val) #1 {
; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
; IR-ITERATIVE: 10:
; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP23:%.*]] syncscope("one-as") monotonic, align 8
; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
; IR-ITERATIVE: 12:
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = phi double [ poison, [[COMPUTEEND:%.*]] ], [ [[TMP11]], [[TMP10:%.*]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP13]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP14]], double [[TMP22:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP13]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP14]], double [[TMP22:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = select i1 [[TMP28:%.*]], double [[TMP14]], double [[TMP15]]
; IR-ITERATIVE-NEXT: br label [[TMP17]]
; IR-ITERATIVE: 17:
@@ -1197,11 +1254,11 @@ define amdgpu_ps double @global_atomic_fadd_double_uni_address_div_value_one_as_
; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi double [ -0.000000e+00, [[TMP2]] ], [ [[TMP23]], [[COMPUTELOOP]] ]
; IR-ITERATIVE-NEXT: [[OLDVALUEPHI:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP22]], [[COMPUTELOOP]] ]
; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP26:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = trunc i64 [[TMP19]] to i32
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP22]] = call double @llvm.amdgcn.writelane.f64(double [[ACCUMULATOR]], i32 [[TMP20]], double [[OLDVALUEPHI]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP23]] = call double @llvm.experimental.constrained.fadd.f64(double [[ACCUMULATOR]], double [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP22]] = call double @llvm.amdgcn.writelane.f64(double [[ACCUMULATOR]], i32 [[TMP20]], double [[OLDVALUEPHI]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP23]] = call double @llvm.experimental.constrained.fadd.f64(double [[ACCUMULATOR]], double [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = shl i64 1, [[TMP19]]
; IR-ITERATIVE-NEXT: [[TMP25:%.*]] = xor i64 [[TMP24]], -1
; IR-ITERATIVE-NEXT: [[TMP26]] = and i64 [[ACTIVEBITS]], [[TMP25]]
@@ -1212,31 +1269,31 @@ define amdgpu_ps double @global_atomic_fadd_double_uni_address_div_value_one_as_
; IR-ITERATIVE-NEXT: br i1 [[TMP28]], label [[TMP10]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP34:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.set.inactive.f64(double [[VAL:%.*]], double -0.000000e+00) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP9]], double [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP13:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP11]], double [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP13]], double [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP15]], double [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP17]], double [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP19]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[TMP21]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP24:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP23]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.set.inactive.f64(double [[VAL:%.*]], double -0.000000e+00) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP10:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP9]], double [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP13:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP11]], double [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP13]], double [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP17:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP15]], double [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP17]], double [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP19]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP22:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[TMP21]], i32 63) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP24:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP23]]) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP25:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP28:%.*]]
; IR-DPP: 26:
@@ -1244,9 +1301,9 @@ define amdgpu_ps double @global_atomic_fadd_double_uni_address_div_value_one_as_
; IR-DPP-NEXT: br label [[TMP28]]
; IR-DPP: 28:
; IR-DPP-NEXT: [[TMP29:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP27]], [[TMP26]] ]
-; IR-DPP-NEXT: [[TMP30:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP29]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP31:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP22]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP32:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP30]], double [[TMP31]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP30:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP29]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP31:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP22]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP32:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP30]], double [[TMP31]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-DPP-NEXT: [[TMP33:%.*]] = select i1 [[TMP25]], double [[TMP30]], double [[TMP32]]
; IR-DPP-NEXT: br label [[TMP34]]
; IR-DPP: 34:
@@ -1259,64 +1316,58 @@ define amdgpu_ps double @global_atomic_fadd_double_uni_address_div_value_one_as_
define amdgpu_ps double @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #2 {
; IR-ITERATIVE-LABEL: @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 1) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-ITERATIVE: 14:
+; IR-ITERATIVE: 11:
; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 8
; IR-ITERATIVE-NEXT: br label [[TMP16]]
-; IR-ITERATIVE: 16:
+; IR-ITERATIVE: 13:
; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP17]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP18]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP17]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP18]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], double [[TMP18]], double [[TMP21]]
; IR-ITERATIVE-NEXT: br label [[TMP23]]
-; IR-ITERATIVE: 23:
+; IR-ITERATIVE: 20:
; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ]
; IR-ITERATIVE-NEXT: ret double [[TMP24]]
;
; IR-DPP-LABEL: @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-DPP: 14:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 8
; IR-DPP-NEXT: br label [[TMP16]]
-; IR-DPP: 16:
+; IR-DPP: 13:
; IR-DPP-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP17]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP18]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP17]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP18]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-DPP-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], double [[TMP18]], double [[TMP21]]
; IR-DPP-NEXT: br label [[TMP23]]
-; IR-DPP: 23:
+; IR-DPP: 20:
; IR-DPP-NEXT: [[TMP24:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ]
; IR-DPP-NEXT: ret double [[TMP24]]
;
@@ -1326,24 +1377,24 @@ define amdgpu_ps double @global_atomic_fsub_double_uni_address_uni_value_agent_s
define amdgpu_ps double @global_atomic_fsub_double_uni_address_div_value_agent_scope_strictfp(ptr addrspace(1) inreg %ptr, double %val) #2 {
; IR-ITERATIVE-LABEL: @global_atomic_fsub_double_uni_address_div_value_agent_scope_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
; IR-ITERATIVE: 10:
; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], double [[TMP23:%.*]] syncscope("agent") monotonic, align 8
; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
; IR-ITERATIVE: 12:
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = phi double [ poison, [[COMPUTEEND:%.*]] ], [ [[TMP11]], [[TMP10:%.*]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP13]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.fsub.f64(double [[TMP14]], double [[TMP22:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP13]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.fsub.f64(double [[TMP14]], double [[TMP22:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = select i1 [[TMP28:%.*]], double [[TMP14]], double [[TMP15]]
; IR-ITERATIVE-NEXT: br label [[TMP17]]
; IR-ITERATIVE: 17:
@@ -1353,11 +1404,11 @@ define amdgpu_ps double @global_atomic_fsub_double_uni_address_div_value_agent_s
; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi double [ -0.000000e+00, [[TMP2]] ], [ [[TMP23]], [[COMPUTELOOP]] ]
; IR-ITERATIVE-NEXT: [[OLDVALUEPHI:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP22]], [[COMPUTELOOP]] ]
; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP26:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = trunc i64 [[TMP19]] to i32
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP22]] = call double @llvm.amdgcn.writelane.f64(double [[ACCUMULATOR]], i32 [[TMP20]], double [[OLDVALUEPHI]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP23]] = call double @llvm.experimental.constrained.fadd.f64(double [[ACCUMULATOR]], double [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP22]] = call double @llvm.amdgcn.writelane.f64(double [[ACCUMULATOR]], i32 [[TMP20]], double [[OLDVALUEPHI]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP23]] = call double @llvm.experimental.constrained.fadd.f64(double [[ACCUMULATOR]], double [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = shl i64 1, [[TMP19]]
; IR-ITERATIVE-NEXT: [[TMP25:%.*]] = xor i64 [[TMP24]], -1
; IR-ITERATIVE-NEXT: [[TMP26]] = and i64 [[ACTIVEBITS]], [[TMP25]]
@@ -1368,31 +1419,31 @@ define amdgpu_ps double @global_atomic_fsub_double_uni_address_div_value_agent_s
; IR-ITERATIVE-NEXT: br i1 [[TMP28]], label [[TMP10]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fsub_double_uni_address_div_value_agent_scope_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP34:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.set.inactive.f64(double [[VAL:%.*]], double -0.000000e+00) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP9]], double [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP13:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP11]], double [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP13]], double [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP15]], double [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP17]], double [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP19]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[TMP21]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP24:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP23]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.set.inactive.f64(double [[VAL:%.*]], double -0.000000e+00) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP10:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP9]], double [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP13:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP11]], double [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP13]], double [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP17:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP15]], double [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP17]], double [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP19]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP22:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[TMP21]], i32 63) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP24:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP23]]) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP25:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP28:%.*]]
; IR-DPP: 26:
@@ -1400,9 +1451,9 @@ define amdgpu_ps double @global_atomic_fsub_double_uni_address_div_value_agent_s
; IR-DPP-NEXT: br label [[TMP28]]
; IR-DPP: 28:
; IR-DPP-NEXT: [[TMP29:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP27]], [[TMP26]] ]
-; IR-DPP-NEXT: [[TMP30:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP29]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP31:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP22]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP32:%.*]] = call double @llvm.experimental.constrained.fsub.f64(double [[TMP30]], double [[TMP31]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP30:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP29]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP31:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP22]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP32:%.*]] = call double @llvm.experimental.constrained.fsub.f64(double [[TMP30]], double [[TMP31]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-DPP-NEXT: [[TMP33:%.*]] = select i1 [[TMP25]], double [[TMP30]], double [[TMP32]]
; IR-DPP-NEXT: br label [[TMP34]]
; IR-DPP: 34:
@@ -1414,32 +1465,61 @@ define amdgpu_ps double @global_atomic_fsub_double_uni_address_div_value_agent_s
}
define amdgpu_ps double @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double inreg %val) #0 {
-; IR-LABEL: @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe(
-; IR-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
-; IR-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP19:%.*]]
-; IR: 2:
-; IR-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
-; IR-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
-; IR-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
-; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
-; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
-; IR-NEXT: [[TMP11:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8
-; IR-NEXT: br label [[TMP12]]
-; IR: 12:
-; IR-NEXT: [[TMP13:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP11]], [[TMP10]] ]
-; IR-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP13]])
-; IR-NEXT: [[TMP15:%.*]] = uitofp i32 [[TMP8]] to double
-; IR-NEXT: [[TMP16:%.*]] = select i1 [[TMP9]], double +qnan, double [[VAL]]
-; IR-NEXT: [[TMP17:%.*]] = call double @llvm.minnum.f64(double [[TMP14]], double [[TMP16]])
-; IR-NEXT: [[TMP18:%.*]] = select i1 [[TMP9]], double [[TMP14]], double [[TMP17]]
-; IR-NEXT: br label [[TMP19]]
-; IR: 19:
-; IR-NEXT: [[TMP20:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP18]], [[TMP12]] ]
-; IR-NEXT: ret double [[TMP20]]
+; IR-ITERATIVE-LABEL: @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe(
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
+; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP20:%.*]]
+; IR-ITERATIVE: 2:
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.wave.reduce.fmin.f64(double [[VAL:%.*]], i32 1)
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[TMP9]] syncscope("agent") monotonic, align 8
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP12]], [[TMP11]] ]
+; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP14]])
+; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = uitofp i32 [[TMP8]] to double
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = select i1 [[TMP10]], double +qnan, double [[VAL]]
+; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call double @llvm.minnum.f64(double [[TMP15]], double [[TMP17]])
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = select i1 [[TMP10]], double [[TMP15]], double [[TMP18]]
+; IR-ITERATIVE-NEXT: br label [[TMP20]]
+; IR-ITERATIVE: 20:
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP19]], [[TMP13]] ]
+; IR-ITERATIVE-NEXT: ret double [[TMP21]]
+;
+; IR-DPP-LABEL: @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe(
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
+; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP20:%.*]]
+; IR-DPP: 2:
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
+; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.wave.reduce.fmin.f64(double [[VAL:%.*]], i32 2)
+; IR-DPP-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-DPP-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]
+; IR-DPP: 11:
+; IR-DPP-NEXT: [[TMP12:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[TMP9]] syncscope("agent") monotonic, align 8
+; IR-DPP-NEXT: br label [[TMP13]]
+; IR-DPP: 13:
+; IR-DPP-NEXT: [[TMP14:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP12]], [[TMP11]] ]
+; IR-DPP-NEXT: [[TMP15:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP14]])
+; IR-DPP-NEXT: [[TMP16:%.*]] = uitofp i32 [[TMP8]] to double
+; IR-DPP-NEXT: [[TMP17:%.*]] = select i1 [[TMP10]], double +qnan, double [[VAL]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.minnum.f64(double [[TMP15]], double [[TMP17]])
+; IR-DPP-NEXT: [[TMP19:%.*]] = select i1 [[TMP10]], double [[TMP15]], double [[TMP18]]
+; IR-DPP-NEXT: br label [[TMP20]]
+; IR-DPP: 20:
+; IR-DPP-NEXT: [[TMP21:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP19]], [[TMP13]] ]
+; IR-DPP-NEXT: ret double [[TMP21]]
;
%result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic
ret double %result
@@ -1536,56 +1616,58 @@ define amdgpu_ps double @global_atomic_fmin_double_uni_address_div_value_agent_s
define amdgpu_ps double @global_atomic__fmax_double_uni_address_uni_value_agent_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #1{
; IR-ITERATIVE-LABEL: @global_atomic__fmax_double_uni_address_uni_value_agent_scope_unsafe_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP19:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[VAL:%.*]] = call double @llvm.amdgcn.wave.reduce.fmax.f64(double [[VAL1:%.*]], i32 1) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR-ITERATIVE: 10:
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL]] syncscope("agent") monotonic, align 8
; IR-ITERATIVE-NEXT: br label [[TMP12]]
-; IR-ITERATIVE: 12:
+; IR-ITERATIVE: 13:
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP11]], [[TMP10]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP13]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = select i1 [[TMP9]], double +qnan, double [[VAL]]
-; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP14]], double [[TMP16]], metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP13]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = select i1 [[TMP9]], double +qnan, double [[VAL1]]
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP14]], double [[TMP21]], metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = select i1 [[TMP9]], double [[TMP14]], double [[TMP17]]
; IR-ITERATIVE-NEXT: br label [[TMP19]]
-; IR-ITERATIVE: 19:
+; IR-ITERATIVE: 20:
; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP18]], [[TMP12]] ]
; IR-ITERATIVE-NEXT: ret double [[TMP20]]
;
; IR-DPP-LABEL: @global_atomic__fmax_double_uni_address_uni_value_agent_scope_unsafe_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP19:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[VAL:%.*]] = call double @llvm.amdgcn.wave.reduce.fmax.f64(double [[VAL1:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR-DPP: 10:
-; IR-DPP-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8
+; IR-DPP: 11:
+; IR-DPP-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL]] syncscope("agent") monotonic, align 8
; IR-DPP-NEXT: br label [[TMP12]]
-; IR-DPP: 12:
+; IR-DPP: 13:
; IR-DPP-NEXT: [[TMP13:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP11]], [[TMP10]] ]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP13]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = select i1 [[TMP9]], double +qnan, double [[VAL]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP14]], double [[TMP16]], metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP13]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP16:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP21:%.*]] = select i1 [[TMP9]], double +qnan, double [[VAL1]]
+; IR-DPP-NEXT: [[TMP17:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP14]], double [[TMP21]], metadata !"fpexcept.strict") #[[ATTR2]]
; IR-DPP-NEXT: [[TMP18:%.*]] = select i1 [[TMP9]], double [[TMP14]], double [[TMP17]]
; IR-DPP-NEXT: br label [[TMP19]]
-; IR-DPP: 19:
+; IR-DPP: 20:
; IR-DPP-NEXT: [[TMP20:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP18]], [[TMP12]] ]
; IR-DPP-NEXT: ret double [[TMP20]]
;
@@ -1595,24 +1677,24 @@ define amdgpu_ps double @global_atomic__fmax_double_uni_address_uni_value_agent_
define amdgpu_ps double @global_atomic__fmax_double_uni_address_div_value_agent_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, double %val) #1{
; IR-ITERATIVE-LABEL: @global_atomic__fmax_double_uni_address_div_value_agent_scope_unsafe_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
; IR-ITERATIVE: 10:
; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[TMP23:%.*]] syncscope("agent") monotonic, align 8
; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
; IR-ITERATIVE: 12:
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = phi double [ poison, [[COMPUTEEND:%.*]] ], [ [[TMP11]], [[TMP10:%.*]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP13]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP14]], double [[TMP22:%.*]], metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP13]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP14]], double [[TMP22:%.*]], metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = select i1 [[TMP28:%.*]], double [[TMP14]], double [[TMP15]]
; IR-ITERATIVE-NEXT: br label [[TMP17]]
; IR-ITERATIVE: 17:
@@ -1622,11 +1704,11 @@ define amdgpu_ps double @global_atomic__fmax_double_uni_address_div_value_agent_
; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi double [ +qnan, [[TMP2]] ], [ [[TMP23]], [[COMPUTELOOP]] ]
; IR-ITERATIVE-NEXT: [[OLDVALUEPHI:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP22]], [[COMPUTELOOP]] ]
; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP26:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = trunc i64 [[TMP19]] to i32
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP22]] = call double @llvm.amdgcn.writelane.f64(double [[ACCUMULATOR]], i32 [[TMP20]], double [[OLDVALUEPHI]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP23]] = call double @llvm.experimental.constrained.maxnum.f64(double [[ACCUMULATOR]], double [[TMP21]], metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP22]] = call double @llvm.amdgcn.writelane.f64(double [[ACCUMULATOR]], i32 [[TMP20]], double [[OLDVALUEPHI]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP23]] = call double @llvm.experimental.constrained.maxnum.f64(double [[ACCUMULATOR]], double [[TMP21]], metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = shl i64 1, [[TMP19]]
; IR-ITERATIVE-NEXT: [[TMP25:%.*]] = xor i64 [[TMP24]], -1
; IR-ITERATIVE-NEXT: [[TMP26]] = and i64 [[ACTIVEBITS]], [[TMP25]]
@@ -1637,31 +1719,31 @@ define amdgpu_ps double @global_atomic__fmax_double_uni_address_div_value_agent_
; IR-ITERATIVE-NEXT: br i1 [[TMP28]], label [[TMP10]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic__fmax_double_uni_address_div_value_agent_scope_unsafe_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP34:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.set.inactive.f64(double [[VAL:%.*]], double +qnan) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP9]], double [[TMP10]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP13:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP11]], double [[TMP12]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP13]], double [[TMP14]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP15]], double [[TMP16]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP17]], double [[TMP18]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP19]], double [[TMP20]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[TMP21]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP24:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP23]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.set.inactive.f64(double [[VAL:%.*]], double +qnan) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP10:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP9]], double [[TMP10]], metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP13:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP11]], double [[TMP12]], metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP13]], double [[TMP14]], metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP17:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP15]], double [[TMP16]], metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP17]], double [[TMP18]], metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP19]], double [[TMP20]], metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP22:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[TMP21]], i32 63) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP24:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP23]]) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP25:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP28:%.*]]
; IR-DPP: 26:
@@ -1669,9 +1751,9 @@ define amdgpu_ps double @global_atomic__fmax_double_uni_address_div_value_agent_
; IR-DPP-NEXT: br label [[TMP28]]
; IR-DPP: 28:
; IR-DPP-NEXT: [[TMP29:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP27]], [[TMP26]] ]
-; IR-DPP-NEXT: [[TMP30:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP29]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP31:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP22]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP32:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP30]], double [[TMP31]], metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP30:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP29]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP31:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP22]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP32:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP30]], double [[TMP31]], metadata !"fpexcept.strict") #[[ATTR2]]
; IR-DPP-NEXT: [[TMP33:%.*]] = select i1 [[TMP25]], double [[TMP30]], double [[TMP32]]
; IR-DPP-NEXT: br label [[TMP34]]
; IR-DPP: 34:
@@ -1684,64 +1766,58 @@ define amdgpu_ps double @global_atomic__fmax_double_uni_address_div_value_agent_
define amdgpu_ps double @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #2 {
; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 1) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-ITERATIVE: 14:
+; IR-ITERATIVE: 11:
; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] monotonic, align 4
; IR-ITERATIVE-NEXT: br label [[TMP16]]
-; IR-ITERATIVE: 16:
+; IR-ITERATIVE: 13:
; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP17]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP18]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP17]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP18]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], double [[TMP18]], double [[TMP21]]
; IR-ITERATIVE-NEXT: br label [[TMP23]]
-; IR-ITERATIVE: 23:
+; IR-ITERATIVE: 20:
; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ]
; IR-ITERATIVE-NEXT: ret double [[TMP24]]
;
; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-DPP: 14:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] monotonic, align 4
; IR-DPP-NEXT: br label [[TMP16]]
-; IR-DPP: 16:
+; IR-DPP: 13:
; IR-DPP-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP17]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP18]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP17]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP18]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-DPP-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], double [[TMP18]], double [[TMP21]]
; IR-DPP-NEXT: br label [[TMP23]]
-; IR-DPP: 23:
+; IR-DPP: 20:
; IR-DPP-NEXT: [[TMP24:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ]
; IR-DPP-NEXT: ret double [[TMP24]]
;
@@ -1751,24 +1827,24 @@ define amdgpu_ps double @global_atomic_fadd_double_uni_address_uni_value_system_
define amdgpu_ps double @global_atomic_fadd_double_uni_address_div_value_system_scope_strictfp(ptr addrspace(1) inreg %ptr, double %val) #2 {
; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_div_value_system_scope_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
; IR-ITERATIVE: 10:
; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP23:%.*]] monotonic, align 4
; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
; IR-ITERATIVE: 12:
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = phi double [ poison, [[COMPUTEEND:%.*]] ], [ [[TMP11]], [[TMP10:%.*]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP13]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP14]], double [[TMP22:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP13]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP14]], double [[TMP22:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = select i1 [[TMP28:%.*]], double [[TMP14]], double [[TMP15]]
; IR-ITERATIVE-NEXT: br label [[TMP17]]
; IR-ITERATIVE: 17:
@@ -1778,11 +1854,11 @@ define amdgpu_ps double @global_atomic_fadd_double_uni_address_div_value_system_
; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi double [ -0.000000e+00, [[TMP2]] ], [ [[TMP23]], [[COMPUTELOOP]] ]
; IR-ITERATIVE-NEXT: [[OLDVALUEPHI:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP22]], [[COMPUTELOOP]] ]
; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP26:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = trunc i64 [[TMP19]] to i32
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP22]] = call double @llvm.amdgcn.writelane.f64(double [[ACCUMULATOR]], i32 [[TMP20]], double [[OLDVALUEPHI]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP23]] = call double @llvm.experimental.constrained.fadd.f64(double [[ACCUMULATOR]], double [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP22]] = call double @llvm.amdgcn.writelane.f64(double [[ACCUMULATOR]], i32 [[TMP20]], double [[OLDVALUEPHI]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP23]] = call double @llvm.experimental.constrained.fadd.f64(double [[ACCUMULATOR]], double [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = shl i64 1, [[TMP19]]
; IR-ITERATIVE-NEXT: [[TMP25:%.*]] = xor i64 [[TMP24]], -1
; IR-ITERATIVE-NEXT: [[TMP26]] = and i64 [[ACTIVEBITS]], [[TMP25]]
@@ -1793,31 +1869,31 @@ define amdgpu_ps double @global_atomic_fadd_double_uni_address_div_value_system_
; IR-ITERATIVE-NEXT: br i1 [[TMP28]], label [[TMP10]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_div_value_system_scope_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP34:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.set.inactive.f64(double [[VAL:%.*]], double -0.000000e+00) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP9]], double [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP13:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP11]], double [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP13]], double [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP15]], double [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP17]], double [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP19]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[TMP21]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP24:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP23]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.set.inactive.f64(double [[VAL:%.*]], double -0.000000e+00) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP10:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP9]], double [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP13:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP11]], double [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP13]], double [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP17:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP15]], double [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP17]], double [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP19]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP22:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[TMP21]], i32 63) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP24:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP23]]) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP25:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP28:%.*]]
; IR-DPP: 26:
@@ -1825,9 +1901,9 @@ define amdgpu_ps double @global_atomic_fadd_double_uni_address_div_value_system_
; IR-DPP-NEXT: br label [[TMP28]]
; IR-DPP: 28:
; IR-DPP-NEXT: [[TMP29:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP27]], [[TMP26]] ]
-; IR-DPP-NEXT: [[TMP30:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP29]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP31:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP22]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP32:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP30]], double [[TMP31]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP30:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP29]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP31:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP22]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP32:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP30]], double [[TMP31]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-DPP-NEXT: [[TMP33:%.*]] = select i1 [[TMP25]], double [[TMP30]], double [[TMP32]]
; IR-DPP-NEXT: br label [[TMP34]]
; IR-DPP: 34:
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan.ll
index b1ff40d64d224..d9b7c85ae90b2 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan.ll
@@ -22,15 +22,13 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1), ptr addrspace(1) %val
; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)
; IR-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])
-; IR-NEXT: [[TMP7:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP1]])
-; IR-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i32
-; IR-NEXT: [[TMP9:%.*]] = mul i32 [[VALUE]], [[TMP8]]
+; IR-NEXT: [[TMP9:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[VALUE]], i32 1)
; IR-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP6]], 0
; IR-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]
-; IR: 11:
+; IR: 9:
; IR-NEXT: [[TMP12:%.*]] = atomicrmw volatile add ptr addrspace(1) [[GEP]], i32 [[TMP9]] seq_cst, align 4
; IR-NEXT: br label [[TMP13]]
-; IR: 13:
+; IR: 11:
; IR-NEXT: ret void
;
entry:
@@ -71,28 +69,14 @@ define amdgpu_kernel void @divergent_value(ptr addrspace(1) %out, ptr addrspace(
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-NEXT: br label [[COMPUTELOOP:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile add ptr addrspace(1) [[GEP]], i32 [[TMP13:%.*]] seq_cst, align 4
-; IR-NEXT: br label [[TMP9:%.*]]
-; IR: 9:
+; IR-NEXT: [[TMP13:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[VALUE]], i32 1)
+; IR-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP7]], label [[TMP10:%.*]], label [[TMP9:%.*]]
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile add ptr addrspace(1) [[GEP]], i32 [[TMP13]] seq_cst, align 4
+; IR-NEXT: br label [[TMP9]]
+; IR: 10:
; IR-NEXT: ret void
-; IR: ComputeLoop:
-; IR-NEXT: [[ACCUMULATOR:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP13]], [[COMPUTELOOP]] ]
-; IR-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP6]], [[ENTRY]] ], [ [[TMP16:%.*]], [[COMPUTELOOP]] ]
-; IR-NEXT: [[TMP10:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true)
-; IR-NEXT: [[TMP11:%.*]] = trunc i64 [[TMP10]] to i32
-; IR-NEXT: [[TMP12:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[VALUE]], i32 [[TMP11]])
-; IR-NEXT: [[TMP13]] = add i32 [[ACCUMULATOR]], [[TMP12]]
-; IR-NEXT: [[TMP14:%.*]] = shl i64 1, [[TMP10]]
-; IR-NEXT: [[TMP15:%.*]] = xor i64 [[TMP14]], -1
-; IR-NEXT: [[TMP16]] = and i64 [[ACTIVEBITS]], [[TMP15]]
-; IR-NEXT: [[TMP17:%.*]] = icmp eq i64 [[TMP16]], 0
-; IR-NEXT: br i1 [[TMP17]], label [[COMPUTEEND:%.*]], label [[COMPUTELOOP]]
-; IR: ComputeEnd:
-; IR-NEXT: [[TMP18:%.*]] = icmp eq i32 [[TMP5]], 0
-; IR-NEXT: br i1 [[TMP18]], label [[TMP7:%.*]], label [[TMP9]]
;
entry:
%divergent_value.kernarg.segment = call nonnull align 16 dereferenceable(52) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan_fp.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan_fp.ll
index 1aa0da2f696ac..402a468c2599c 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan_fp.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan_fp.ll
@@ -11,16 +11,13 @@ define amdgpu_kernel void @global_atomic_fadd_uni_value(ptr addrspace(1) %ptr) {
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP1]])
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i32
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = uitofp i32 [[TMP8]] to float
-; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = fmul float 4.000000e+00, [[TMP9]]
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float 4.000000e+00, i32 1)
; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = icmp eq i32 [[TMP6]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP14:%.*]]
-; IR-ITERATIVE: 12:
+; IR-ITERATIVE: 9:
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP10]] seq_cst, align 4
; IR-ITERATIVE-NEXT: br label [[TMP14]]
-; IR-ITERATIVE: 14:
+; IR-ITERATIVE: 11:
; IR-ITERATIVE-NEXT: ret void
;
; IR-DPP-LABEL: @global_atomic_fadd_uni_value(
@@ -30,16 +27,13 @@ define amdgpu_kernel void @global_atomic_fadd_uni_value(ptr addrspace(1) %ptr) {
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)
; IR-DPP-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP1]])
-; IR-DPP-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i32
-; IR-DPP-NEXT: [[TMP9:%.*]] = uitofp i32 [[TMP8]] to float
-; IR-DPP-NEXT: [[TMP10:%.*]] = fmul float 4.000000e+00, [[TMP9]]
+; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float 4.000000e+00, i32 2)
; IR-DPP-NEXT: [[TMP11:%.*]] = icmp eq i32 [[TMP6]], 0
; IR-DPP-NEXT: br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP14:%.*]]
-; IR-DPP: 12:
+; IR-DPP: 9:
; IR-DPP-NEXT: [[TMP13:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP10]] seq_cst, align 4
; IR-DPP-NEXT: br label [[TMP14]]
-; IR-DPP: 14:
+; IR-DPP: 11:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 seq_cst
@@ -57,28 +51,14 @@ define amdgpu_kernel void @global_atomic_fadd_div_value(ptr addrspace(1) %ptr) {
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
-; IR-ITERATIVE: 8:
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP14:%.*]] seq_cst, align 4
-; IR-ITERATIVE-NEXT: br label [[TMP10:%.*]]
-; IR-ITERATIVE: 10:
+; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[DIVVALUE]], i32 1)
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP6]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP8]], label [[TMP11:%.*]], label [[TMP10:%.*]]
+; IR-ITERATIVE: 9:
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP14]] seq_cst, align 4
+; IR-ITERATIVE-NEXT: br label [[TMP10]]
+; IR-ITERATIVE: 11:
; IR-ITERATIVE-NEXT: ret void
-; IR-ITERATIVE: ComputeLoop:
-; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ -0.000000e+00, [[TMP0:%.*]] ], [ [[TMP14]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP7]], [[TMP0]] ], [ [[TMP17:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true)
-; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = trunc i64 [[TMP11]] to i32
-; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[DIVVALUE]], i32 [[TMP12]])
-; IR-ITERATIVE-NEXT: [[TMP14]] = fadd float [[ACCUMULATOR]], [[TMP13]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = shl i64 1, [[TMP11]]
-; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = xor i64 [[TMP15]], -1
-; IR-ITERATIVE-NEXT: [[TMP17]] = and i64 [[ACTIVEBITS]], [[TMP16]]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = icmp eq i64 [[TMP17]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP18]], label [[COMPUTEEND:%.*]], label [[COMPUTELOOP]]
-; IR-ITERATIVE: ComputeEnd:
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = icmp eq i32 [[TMP6]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP19]], label [[TMP8:%.*]], label [[TMP10]]
;
; IR-DPP-LABEL: @global_atomic_fadd_div_value(
; IR-DPP-NEXT: [[ID_X:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
@@ -89,27 +69,13 @@ define amdgpu_kernel void @global_atomic_fadd_div_value(ptr addrspace(1) %ptr) {
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)
; IR-DPP-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])
-; IR-DPP-NEXT: [[TMP7:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[DIVVALUE]], float -0.000000e+00)
-; IR-DPP-NEXT: [[TMP8:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP7]], i32 273, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP9:%.*]] = fadd float [[TMP7]], [[TMP8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP9]], i32 274, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP11:%.*]] = fadd float [[TMP9]], [[TMP10]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP11]], i32 276, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP13:%.*]] = fadd float [[TMP11]], [[TMP12]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP13]], i32 280, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP15:%.*]] = fadd float [[TMP13]], [[TMP14]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP15]], i32 322, i32 10, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP17:%.*]] = fadd float [[TMP15]], [[TMP16]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP17]], i32 323, i32 12, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP19:%.*]] = fadd float [[TMP17]], [[TMP18]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP19]], i32 63)
-; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP20]])
+; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[DIVVALUE]], i32 2)
; IR-DPP-NEXT: [[TMP22:%.*]] = icmp eq i32 [[TMP6]], 0
; IR-DPP-NEXT: br i1 [[TMP22]], label [[TMP23:%.*]], label [[TMP25:%.*]]
-; IR-DPP: 23:
+; IR-DPP: 9:
; IR-DPP-NEXT: [[TMP24:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP21]] seq_cst, align 4
; IR-DPP-NEXT: br label [[TMP25]]
-; IR-DPP: 25:
+; IR-DPP: 11:
; IR-DPP-NEXT: ret void
;
%id.x = call i32 @llvm.amdgcn.workitem.id.x()
@@ -126,16 +92,13 @@ define amdgpu_kernel void @global_atomic_fsub_uni_value(ptr addrspace(1) %ptr) {
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP1]])
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i32
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = uitofp i32 [[TMP8]] to float
-; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = fmul float 4.000000e+00, [[TMP9]]
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float 4.000000e+00, i32 1)
; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = icmp eq i32 [[TMP6]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP14:%.*]]
-; IR-ITERATIVE: 12:
+; IR-ITERATIVE: 9:
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP10]] seq_cst, align 4
; IR-ITERATIVE-NEXT: br label [[TMP14]]
-; IR-ITERATIVE: 14:
+; IR-ITERATIVE: 11:
; IR-ITERATIVE-NEXT: ret void
;
; IR-DPP-LABEL: @global_atomic_fsub_uni_value(
@@ -145,16 +108,13 @@ define amdgpu_kernel void @global_atomic_fsub_uni_value(ptr addrspace(1) %ptr) {
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)
; IR-DPP-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP1]])
-; IR-DPP-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i32
-; IR-DPP-NEXT: [[TMP9:%.*]] = uitofp i32 [[TMP8]] to float
-; IR-DPP-NEXT: [[TMP10:%.*]] = fmul float 4.000000e+00, [[TMP9]]
+; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float 4.000000e+00, i32 2)
; IR-DPP-NEXT: [[TMP11:%.*]] = icmp eq i32 [[TMP6]], 0
; IR-DPP-NEXT: br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP14:%.*]]
-; IR-DPP: 12:
+; IR-DPP: 9:
; IR-DPP-NEXT: [[TMP13:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP10]] seq_cst, align 4
; IR-DPP-NEXT: br label [[TMP14]]
-; IR-DPP: 14:
+; IR-DPP: 11:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 seq_cst
@@ -172,28 +132,14 @@ define amdgpu_kernel void @global_atomic_fsub_div_value(ptr addrspace(1) %ptr) {
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
-; IR-ITERATIVE: 8:
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], float [[TMP14:%.*]] seq_cst, align 4
-; IR-ITERATIVE-NEXT: br label [[TMP10:%.*]]
-; IR-ITERATIVE: 10:
+; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[DIVVALUE]], i32 1)
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP6]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP8]], label [[TMP11:%.*]], label [[TMP10:%.*]]
+; IR-ITERATIVE: 9:
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], float [[TMP14]] seq_cst, align 4
+; IR-ITERATIVE-NEXT: br label [[TMP10]]
+; IR-ITERATIVE: 11:
; IR-ITERATIVE-NEXT: ret void
-; IR-ITERATIVE: ComputeLoop:
-; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ -0.000000e+00, [[TMP0:%.*]] ], [ [[TMP14]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP7]], [[TMP0]] ], [ [[TMP17:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true)
-; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = trunc i64 [[TMP11]] to i32
-; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[DIVVALUE]], i32 [[TMP12]])
-; IR-ITERATIVE-NEXT: [[TMP14]] = fadd float [[ACCUMULATOR]], [[TMP13]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = shl i64 1, [[TMP11]]
-; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = xor i64 [[TMP15]], -1
-; IR-ITERATIVE-NEXT: [[TMP17]] = and i64 [[ACTIVEBITS]], [[TMP16]]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = icmp eq i64 [[TMP17]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP18]], label [[COMPUTEEND:%.*]], label [[COMPUTELOOP]]
-; IR-ITERATIVE: ComputeEnd:
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = icmp eq i32 [[TMP6]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP19]], label [[TMP8:%.*]], label [[TMP10]]
;
; IR-DPP-LABEL: @global_atomic_fsub_div_value(
; IR-DPP-NEXT: [[ID_X:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
@@ -204,27 +150,13 @@ define amdgpu_kernel void @global_atomic_fsub_div_value(ptr addrspace(1) %ptr) {
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)
; IR-DPP-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])
-; IR-DPP-NEXT: [[TMP7:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[DIVVALUE]], float -0.000000e+00)
-; IR-DPP-NEXT: [[TMP8:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP7]], i32 273, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP9:%.*]] = fadd float [[TMP7]], [[TMP8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP9]], i32 274, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP11:%.*]] = fadd float [[TMP9]], [[TMP10]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP11]], i32 276, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP13:%.*]] = fadd float [[TMP11]], [[TMP12]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP13]], i32 280, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP15:%.*]] = fadd float [[TMP13]], [[TMP14]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP15]], i32 322, i32 10, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP17:%.*]] = fadd float [[TMP15]], [[TMP16]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP17]], i32 323, i32 12, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP19:%.*]] = fadd float [[TMP17]], [[TMP18]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP19]], i32 63)
-; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP20]])
+; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[DIVVALUE]], i32 2)
; IR-DPP-NEXT: [[TMP22:%.*]] = icmp eq i32 [[TMP6]], 0
; IR-DPP-NEXT: br i1 [[TMP22]], label [[TMP23:%.*]], label [[TMP25:%.*]]
-; IR-DPP: 23:
+; IR-DPP: 9:
; IR-DPP-NEXT: [[TMP24:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], float [[TMP21]] seq_cst, align 4
; IR-DPP-NEXT: br label [[TMP25]]
-; IR-DPP: 25:
+; IR-DPP: 11:
; IR-DPP-NEXT: ret void
;
%id.x = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_optimizer_fp_no_rtn.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_optimizer_fp_no_rtn.ll
index bda6511857b11..5cf8adc778061 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_optimizer_fp_no_rtn.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_optimizer_fp_no_rtn.ll
@@ -7,29 +7,47 @@
; strategies are valid for only divergent values. This optimization is valid for divergent addresses. Test also covers different scopes.
define amdgpu_ps void @global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, float inreg %val) #0 {
-; IR-LABEL: @global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe(
-; IR-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
-; IR-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
-; IR: 2:
-; IR-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
-; IR-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
-; IR-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
-; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
-; IR-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]])
-; IR-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-NEXT: [[TMP11:%.*]] = uitofp i32 [[TMP10]] to float
-; IR-NEXT: [[TMP12:%.*]] = fmul float [[VAL:%.*]], [[TMP11]]
-; IR-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR: 14:
-; IR-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] syncscope("agent") monotonic, align 4
-; IR-NEXT: br label [[TMP16]]
-; IR: 16:
-; IR-NEXT: br label [[TMP17]]
-; IR: 17:
-; IR-NEXT: ret void
+; IR-ITERATIVE-LABEL: @global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe(
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
+; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP14:%.*]]
+; IR-ITERATIVE: 2:
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 1)
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP9]] syncscope("agent") monotonic, align 4
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP14]]
+; IR-ITERATIVE: 14:
+; IR-ITERATIVE-NEXT: ret void
+;
+; IR-DPP-LABEL: @global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe(
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
+; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP14:%.*]]
+; IR-DPP: 2:
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
+; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 2)
+; IR-DPP-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-DPP-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]
+; IR-DPP: 11:
+; IR-DPP-NEXT: [[TMP12:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP9]] syncscope("agent") monotonic, align 4
+; IR-DPP-NEXT: br label [[TMP13]]
+; IR-DPP: 13:
+; IR-DPP-NEXT: br label [[TMP14]]
+; IR-DPP: 14:
+; IR-DPP-NEXT: ret void
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, float %val syncscope("agent") monotonic, align 4
ret void
@@ -46,30 +64,16 @@ define amdgpu_ps void @global_atomic_fadd_uni_address_div_value_scope_agent_scop
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
-; IR-ITERATIVE: 10:
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP17:%.*]] syncscope("agent") monotonic, align 4
-; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
-; IR-ITERATIVE: 12:
-; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 1)
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP14:%.*]], label [[TMP12:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP17]] syncscope("agent") monotonic, align 4
+; IR-ITERATIVE-NEXT: br label [[TMP12]]
; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
-; IR-ITERATIVE: ComputeLoop:
-; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ -0.000000e+00, [[TMP2]] ], [ [[TMP17]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP20:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true)
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32
-; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP15]])
-; IR-ITERATIVE-NEXT: [[TMP17]] = fadd float [[ACCUMULATOR]], [[TMP16]]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = shl i64 1, [[TMP14]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = xor i64 [[TMP18]], -1
-; IR-ITERATIVE-NEXT: [[TMP20]] = and i64 [[ACTIVEBITS]], [[TMP19]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = icmp eq i64 [[TMP20]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP21]], label [[COMPUTEEND:%.*]], label [[COMPUTELOOP]]
-; IR-ITERATIVE: ComputeEnd:
-; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP22]], label [[TMP10:%.*]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fadd_uni_address_div_value_scope_agent_scope_unsafe(
; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
@@ -81,29 +85,15 @@ define amdgpu_ps void @global_atomic_fadd_uni_address_div_value_scope_agent_scop
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
-; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[VAL:%.*]], float -0.000000e+00)
-; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP9]], i32 273, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP11:%.*]] = fadd float [[TMP9]], [[TMP10]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP11]], i32 274, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP13:%.*]] = fadd float [[TMP11]], [[TMP12]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP13]], i32 276, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP15:%.*]] = fadd float [[TMP13]], [[TMP14]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP15]], i32 280, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP17:%.*]] = fadd float [[TMP15]], [[TMP16]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP17]], i32 322, i32 10, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP19:%.*]] = fadd float [[TMP17]], [[TMP18]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP19]], i32 323, i32 12, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP21:%.*]] = fadd float [[TMP19]], [[TMP20]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP21]], i32 63)
-; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]])
+; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 2)
; IR-DPP-NEXT: [[TMP24:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP24]], label [[TMP25:%.*]], label [[TMP27:%.*]]
-; IR-DPP: 25:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP26:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP23]] syncscope("agent") monotonic, align 4
; IR-DPP-NEXT: br label [[TMP27]]
-; IR-DPP: 27:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP28]]
-; IR-DPP: 28:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, float %val syncscope("agent") monotonic, align 4
@@ -112,51 +102,45 @@ define amdgpu_ps void @global_atomic_fadd_uni_address_div_value_scope_agent_scop
define amdgpu_ps void @global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, float inreg %val) #1 {
; IR-ITERATIVE-LABEL: @global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7:[0-9]+]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2:[0-9]+]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 1) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-ITERATIVE: 14:
+; IR-ITERATIVE: 11:
; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] syncscope("one-as") monotonic, align 4
; IR-ITERATIVE-NEXT: br label [[TMP16]]
-; IR-ITERATIVE: 16:
+; IR-ITERATIVE: 13:
; IR-ITERATIVE-NEXT: br label [[TMP17]]
-; IR-ITERATIVE: 17:
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
;
; IR-DPP-LABEL: @global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8:[0-9]+]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2:[0-9]+]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-DPP: 14:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] syncscope("one-as") monotonic, align 4
; IR-DPP-NEXT: br label [[TMP16]]
-; IR-DPP: 16:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP17]]
-; IR-DPP: 17:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, float %val syncscope("one-as") monotonic
@@ -165,73 +149,45 @@ define amdgpu_ps void @global_atomic_fadd_uni_address_uni_value_one_as_scope_uns
define amdgpu_ps void @global_atomic_fadd_uni_address_div_value_one_as_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, float %val) #1 {
; IR-ITERATIVE-LABEL: @global_atomic_fadd_uni_address_div_value_one_as_scope_unsafe_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
-; IR-ITERATIVE: 10:
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP17:%.*]] syncscope("one-as") monotonic, align 4
-; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
-; IR-ITERATIVE: 12:
-; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 1) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP14:%.*]], label [[TMP12:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP17]] syncscope("one-as") monotonic, align 4
+; IR-ITERATIVE-NEXT: br label [[TMP12]]
; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
-; IR-ITERATIVE: ComputeLoop:
-; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ -0.000000e+00, [[TMP2]] ], [ [[TMP17]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP20:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32
-; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP15]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP17]] = call float @llvm.experimental.constrained.fadd.f32(float [[ACCUMULATOR]], float [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = shl i64 1, [[TMP14]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = xor i64 [[TMP18]], -1
-; IR-ITERATIVE-NEXT: [[TMP20]] = and i64 [[ACTIVEBITS]], [[TMP19]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = icmp eq i64 [[TMP20]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP21]], label [[COMPUTEEND:%.*]], label [[COMPUTELOOP]]
-; IR-ITERATIVE: ComputeEnd:
-; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP22]], label [[TMP10:%.*]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fadd_uni_address_div_value_one_as_scope_unsafe_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP28:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[VAL:%.*]], float -0.000000e+00) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP9]], float [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP13:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP11]], float [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP13]], float [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP15]], float [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP17]], float [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP19]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP21]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP24:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP24]], label [[TMP25:%.*]], label [[TMP27:%.*]]
-; IR-DPP: 25:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP26:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP23]] syncscope("one-as") monotonic, align 4
; IR-DPP-NEXT: br label [[TMP27]]
-; IR-DPP: 27:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP28]]
-; IR-DPP: 28:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, float %val syncscope("one-as") monotonic
@@ -240,51 +196,45 @@ define amdgpu_ps void @global_atomic_fadd_uni_address_div_value_one_as_scope_uns
define amdgpu_ps void @global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp(ptr addrspace(1) inreg %ptr, float inreg %val) #2 {
; IR-ITERATIVE-LABEL: @global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 1) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-ITERATIVE: 14:
+; IR-ITERATIVE: 11:
; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] syncscope("agent") monotonic, align 4
; IR-ITERATIVE-NEXT: br label [[TMP16]]
-; IR-ITERATIVE: 16:
+; IR-ITERATIVE: 13:
; IR-ITERATIVE-NEXT: br label [[TMP17]]
-; IR-ITERATIVE: 17:
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
;
; IR-DPP-LABEL: @global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-DPP: 14:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] syncscope("agent") monotonic, align 4
; IR-DPP-NEXT: br label [[TMP16]]
-; IR-DPP: 16:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP17]]
-; IR-DPP: 17:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, float %val syncscope("agent") monotonic
@@ -293,73 +243,45 @@ define amdgpu_ps void @global_atomic_fsub_uni_address_uni_value_agent_scope_stri
define amdgpu_ps void @global_atomic_fsub_uni_address_div_value_agent_scope_strictfp(ptr addrspace(1) inreg %ptr, float %val) #2 {
; IR-ITERATIVE-LABEL: @global_atomic_fsub_uni_address_div_value_agent_scope_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
-; IR-ITERATIVE: 10:
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], float [[TMP17:%.*]] syncscope("agent") monotonic, align 4
-; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
-; IR-ITERATIVE: 12:
-; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 1) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP14:%.*]], label [[TMP12:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], float [[TMP17]] syncscope("agent") monotonic, align 4
+; IR-ITERATIVE-NEXT: br label [[TMP12]]
; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
-; IR-ITERATIVE: ComputeLoop:
-; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ -0.000000e+00, [[TMP2]] ], [ [[TMP17]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP20:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32
-; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP15]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP17]] = call float @llvm.experimental.constrained.fadd.f32(float [[ACCUMULATOR]], float [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = shl i64 1, [[TMP14]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = xor i64 [[TMP18]], -1
-; IR-ITERATIVE-NEXT: [[TMP20]] = and i64 [[ACTIVEBITS]], [[TMP19]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = icmp eq i64 [[TMP20]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP21]], label [[COMPUTEEND:%.*]], label [[COMPUTELOOP]]
-; IR-ITERATIVE: ComputeEnd:
-; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP22]], label [[TMP10:%.*]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fsub_uni_address_div_value_agent_scope_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP28:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[VAL:%.*]], float -0.000000e+00) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP9]], float [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP13:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP11]], float [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP13]], float [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP15]], float [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP17]], float [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP19]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP21]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP24:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP24]], label [[TMP25:%.*]], label [[TMP27:%.*]]
-; IR-DPP: 25:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP26:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], float [[TMP23]] syncscope("agent") monotonic, align 4
; IR-DPP-NEXT: br label [[TMP27]]
-; IR-DPP: 27:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP28]]
-; IR-DPP: 28:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fsub ptr addrspace(1) %ptr, float %val syncscope("agent") monotonic
@@ -367,25 +289,47 @@ define amdgpu_ps void @global_atomic_fsub_uni_address_div_value_agent_scope_stri
}
define amdgpu_ps void @global_atomic_fmin_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, float inreg %val) #0 {
-; IR-LABEL: @global_atomic_fmin_uni_address_uni_value_agent_scope_unsafe(
-; IR-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
-; IR-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]]
-; IR: 2:
-; IR-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
-; IR-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
-; IR-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
-; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
-; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
-; IR-NEXT: [[TMP11:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], float [[VAL:%.*]] syncscope("agent") monotonic, align 4
-; IR-NEXT: br label [[TMP12]]
-; IR: 12:
-; IR-NEXT: br label [[TMP13]]
-; IR: 13:
-; IR-NEXT: ret void
+; IR-ITERATIVE-LABEL: @global_atomic_fmin_uni_address_uni_value_agent_scope_unsafe(
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
+; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP14:%.*]]
+; IR-ITERATIVE: 2:
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.wave.reduce.fmin.f32(float [[VAL:%.*]], i32 1)
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], float [[TMP9]] syncscope("agent") monotonic, align 4
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP14]]
+; IR-ITERATIVE: 14:
+; IR-ITERATIVE-NEXT: ret void
+;
+; IR-DPP-LABEL: @global_atomic_fmin_uni_address_uni_value_agent_scope_unsafe(
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
+; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP14:%.*]]
+; IR-DPP: 2:
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
+; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.wave.reduce.fmin.f32(float [[VAL:%.*]], i32 2)
+; IR-DPP-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-DPP-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]
+; IR-DPP: 11:
+; IR-DPP-NEXT: [[TMP12:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], float [[TMP9]] syncscope("agent") monotonic, align 4
+; IR-DPP-NEXT: br label [[TMP13]]
+; IR-DPP: 13:
+; IR-DPP-NEXT: br label [[TMP14]]
+; IR-DPP: 14:
+; IR-DPP-NEXT: ret void
;
%result = atomicrmw fmin ptr addrspace(1) %ptr, float %val syncscope("agent") monotonic
ret void
@@ -402,30 +346,16 @@ define amdgpu_ps void @global_atomic_fmin_uni_address_div_value_agent_scope_unsa
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
-; IR-ITERATIVE: 10:
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], float [[TMP17:%.*]] syncscope("agent") monotonic, align 4
-; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
-; IR-ITERATIVE: 12:
-; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = call float @llvm.amdgcn.wave.reduce.fmin.f32(float [[VAL:%.*]], i32 1)
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP14:%.*]], label [[TMP12:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], float [[TMP17]] syncscope("agent") monotonic, align 4
+; IR-ITERATIVE-NEXT: br label [[TMP12]]
; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
-; IR-ITERATIVE: ComputeLoop:
-; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ +qnan, [[TMP2]] ], [ [[TMP17]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP20:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true)
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32
-; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP15]])
-; IR-ITERATIVE-NEXT: [[TMP17]] = call float @llvm.minnum.f32(float [[ACCUMULATOR]], float [[TMP16]])
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = shl i64 1, [[TMP14]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = xor i64 [[TMP18]], -1
-; IR-ITERATIVE-NEXT: [[TMP20]] = and i64 [[ACTIVEBITS]], [[TMP19]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = icmp eq i64 [[TMP20]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP21]], label [[COMPUTEEND:%.*]], label [[COMPUTELOOP]]
-; IR-ITERATIVE: ComputeEnd:
-; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP22]], label [[TMP10:%.*]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fmin_uni_address_div_value_agent_scope_unsafe(
; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
@@ -437,29 +367,15 @@ define amdgpu_ps void @global_atomic_fmin_uni_address_div_value_agent_scope_unsa
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
-; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[VAL:%.*]], float +qnan)
-; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP9]], i32 273, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.minnum.f32(float [[TMP9]], float [[TMP10]])
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP11]], i32 274, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP13:%.*]] = call float @llvm.minnum.f32(float [[TMP11]], float [[TMP12]])
-; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP13]], i32 276, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.minnum.f32(float [[TMP13]], float [[TMP14]])
-; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP15]], i32 280, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP17:%.*]] = call float @llvm.minnum.f32(float [[TMP15]], float [[TMP16]])
-; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP17]], i32 322, i32 10, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.minnum.f32(float [[TMP17]], float [[TMP18]])
-; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP19]], i32 323, i32 12, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.minnum.f32(float [[TMP19]], float [[TMP20]])
-; IR-DPP-NEXT: [[TMP22:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP21]], i32 63)
-; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]])
+; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.wave.reduce.fmin.f32(float [[VAL:%.*]], i32 2)
; IR-DPP-NEXT: [[TMP24:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP24]], label [[TMP25:%.*]], label [[TMP27:%.*]]
-; IR-DPP: 25:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP26:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], float [[TMP23]] syncscope("agent") monotonic, align 4
; IR-DPP-NEXT: br label [[TMP27]]
-; IR-DPP: 27:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP28]]
-; IR-DPP: 28:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fmin ptr addrspace(1) %ptr, float %val syncscope("agent") monotonic
@@ -468,43 +384,45 @@ define amdgpu_ps void @global_atomic_fmin_uni_address_div_value_agent_scope_unsa
define amdgpu_ps void @global_atomic_fmax_uni_address_uni_value_agent_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, float inreg %val) #1{
; IR-ITERATIVE-LABEL: @global_atomic_fmax_uni_address_uni_value_agent_scope_unsafe_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.wave.reduce.fmax.f32(float [[VAL1:%.*]], i32 1) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR-ITERATIVE: 10:
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], float [[VAL:%.*]] syncscope("agent") monotonic, align 4
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], float [[VAL]] syncscope("agent") monotonic, align 4
; IR-ITERATIVE-NEXT: br label [[TMP12]]
-; IR-ITERATIVE: 12:
-; IR-ITERATIVE-NEXT: br label [[TMP13]]
; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
;
; IR-DPP-LABEL: @global_atomic_fmax_uni_address_uni_value_agent_scope_unsafe_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.wave.reduce.fmax.f32(float [[VAL1:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR-DPP: 10:
-; IR-DPP-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], float [[VAL:%.*]] syncscope("agent") monotonic, align 4
+; IR-DPP: 11:
+; IR-DPP-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], float [[VAL]] syncscope("agent") monotonic, align 4
; IR-DPP-NEXT: br label [[TMP12]]
-; IR-DPP: 12:
-; IR-DPP-NEXT: br label [[TMP13]]
; IR-DPP: 13:
+; IR-DPP-NEXT: br label [[TMP13]]
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fmax ptr addrspace(1) %ptr, float %val syncscope("agent") monotonic
@@ -513,73 +431,45 @@ define amdgpu_ps void @global_atomic_fmax_uni_address_uni_value_agent_scope_unsa
define amdgpu_ps void @global_atomic_fmax_uni_address_div_value_agent_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, float %val) #1{
; IR-ITERATIVE-LABEL: @global_atomic_fmax_uni_address_div_value_agent_scope_unsafe_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
-; IR-ITERATIVE: 10:
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], float [[TMP17:%.*]] syncscope("agent") monotonic, align 4
-; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
-; IR-ITERATIVE: 12:
-; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = call float @llvm.amdgcn.wave.reduce.fmax.f32(float [[VAL:%.*]], i32 1) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP14:%.*]], label [[TMP12:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], float [[TMP17]] syncscope("agent") monotonic, align 4
+; IR-ITERATIVE-NEXT: br label [[TMP12]]
; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
-; IR-ITERATIVE: ComputeLoop:
-; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ +qnan, [[TMP2]] ], [ [[TMP17]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP20:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32
-; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP15]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP17]] = call float @llvm.experimental.constrained.maxnum.f32(float [[ACCUMULATOR]], float [[TMP16]], metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = shl i64 1, [[TMP14]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = xor i64 [[TMP18]], -1
-; IR-ITERATIVE-NEXT: [[TMP20]] = and i64 [[ACTIVEBITS]], [[TMP19]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = icmp eq i64 [[TMP20]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP21]], label [[COMPUTEEND:%.*]], label [[COMPUTELOOP]]
-; IR-ITERATIVE: ComputeEnd:
-; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP22]], label [[TMP10:%.*]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fmax_uni_address_div_value_agent_scope_unsafe_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP28:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[VAL:%.*]], float +qnan) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP9]], float [[TMP10]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP13:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP11]], float [[TMP12]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP13]], float [[TMP14]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP15]], float [[TMP16]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP17]], float [[TMP18]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP19]], float [[TMP20]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP21]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.wave.reduce.fmax.f32(float [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP24:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP24]], label [[TMP25:%.*]], label [[TMP27:%.*]]
-; IR-DPP: 25:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP26:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], float [[TMP23]] syncscope("agent") monotonic, align 4
; IR-DPP-NEXT: br label [[TMP27]]
-; IR-DPP: 27:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP28]]
-; IR-DPP: 28:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fmax ptr addrspace(1) %ptr, float %val syncscope("agent") monotonic
@@ -588,51 +478,45 @@ define amdgpu_ps void @global_atomic_fmax_uni_address_div_value_agent_scope_unsa
define amdgpu_ps void @global_atomic_fadd_uni_address_uni_value_system_scope_strictfp(ptr addrspace(1) inreg %ptr, float inreg %val) #2 {
; IR-ITERATIVE-LABEL: @global_atomic_fadd_uni_address_uni_value_system_scope_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 1) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-ITERATIVE: 14:
+; IR-ITERATIVE: 11:
; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] monotonic, align 4
; IR-ITERATIVE-NEXT: br label [[TMP16]]
-; IR-ITERATIVE: 16:
+; IR-ITERATIVE: 13:
; IR-ITERATIVE-NEXT: br label [[TMP17]]
-; IR-ITERATIVE: 17:
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
;
; IR-DPP-LABEL: @global_atomic_fadd_uni_address_uni_value_system_scope_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-DPP: 14:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] monotonic, align 4
; IR-DPP-NEXT: br label [[TMP16]]
-; IR-DPP: 16:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP17]]
-; IR-DPP: 17:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, float %val monotonic, align 4
@@ -641,73 +525,45 @@ define amdgpu_ps void @global_atomic_fadd_uni_address_uni_value_system_scope_str
define amdgpu_ps void @global_atomic_fadd_uni_address_div_value_system_scope_strictfp(ptr addrspace(1) inreg %ptr, float %val) #2 {
; IR-ITERATIVE-LABEL: @global_atomic_fadd_uni_address_div_value_system_scope_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
-; IR-ITERATIVE: 10:
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP17:%.*]] monotonic, align 4
-; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
-; IR-ITERATIVE: 12:
-; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 1) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP14:%.*]], label [[TMP12:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP17]] monotonic, align 4
+; IR-ITERATIVE-NEXT: br label [[TMP12]]
; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
-; IR-ITERATIVE: ComputeLoop:
-; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ -0.000000e+00, [[TMP2]] ], [ [[TMP17]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP20:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32
-; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP15]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP17]] = call float @llvm.experimental.constrained.fadd.f32(float [[ACCUMULATOR]], float [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = shl i64 1, [[TMP14]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = xor i64 [[TMP18]], -1
-; IR-ITERATIVE-NEXT: [[TMP20]] = and i64 [[ACTIVEBITS]], [[TMP19]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = icmp eq i64 [[TMP20]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP21]], label [[COMPUTEEND:%.*]], label [[COMPUTELOOP]]
-; IR-ITERATIVE: ComputeEnd:
-; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP22]], label [[TMP10:%.*]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fadd_uni_address_div_value_system_scope_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP28:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[VAL:%.*]], float -0.000000e+00) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP9]], float [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP13:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP11]], float [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP13]], float [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP15]], float [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP17]], float [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP19]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP21]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP24:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP24]], label [[TMP25:%.*]], label [[TMP27:%.*]]
-; IR-DPP: 25:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP26:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP23]] monotonic, align 4
; IR-DPP-NEXT: br label [[TMP27]]
-; IR-DPP: 27:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP28]]
-; IR-DPP: 28:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, float %val monotonic, align 4
@@ -823,29 +679,47 @@ define amdgpu_ps void @global_atomic_fadd_div_address_div_value_system_scope_str
}
define amdgpu_ps void @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double inreg %val) #0 {
-; IR-LABEL: @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe(
-; IR-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
-; IR-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
-; IR: 2:
-; IR-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
-; IR-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
-; IR-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
-; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
-; IR-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]])
-; IR-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-NEXT: [[TMP11:%.*]] = uitofp i32 [[TMP10]] to double
-; IR-NEXT: [[TMP12:%.*]] = fmul double [[VAL:%.*]], [[TMP11]]
-; IR-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR: 14:
-; IR-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 4
-; IR-NEXT: br label [[TMP16]]
-; IR: 16:
-; IR-NEXT: br label [[TMP17]]
-; IR: 17:
-; IR-NEXT: ret void
+; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe(
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
+; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP14:%.*]]
+; IR-ITERATIVE: 2:
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 1)
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP9]] syncscope("agent") monotonic, align 4
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP14]]
+; IR-ITERATIVE: 14:
+; IR-ITERATIVE-NEXT: ret void
+;
+; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe(
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
+; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP14:%.*]]
+; IR-DPP: 2:
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
+; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 2)
+; IR-DPP-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-DPP-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]
+; IR-DPP: 11:
+; IR-DPP-NEXT: [[TMP12:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP9]] syncscope("agent") monotonic, align 4
+; IR-DPP-NEXT: br label [[TMP13]]
+; IR-DPP: 13:
+; IR-DPP-NEXT: br label [[TMP14]]
+; IR-DPP: 14:
+; IR-DPP-NEXT: ret void
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic, align 4
ret void
@@ -862,30 +736,16 @@ define amdgpu_ps void @global_atomic_fadd_double_uni_address_div_value_scope_age
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
-; IR-ITERATIVE: 10:
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP17:%.*]] syncscope("agent") monotonic, align 4
-; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
-; IR-ITERATIVE: 12:
-; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 1)
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP14:%.*]], label [[TMP12:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP17]] syncscope("agent") monotonic, align 4
+; IR-ITERATIVE-NEXT: br label [[TMP12]]
; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
-; IR-ITERATIVE: ComputeLoop:
-; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi double [ -0.000000e+00, [[TMP2]] ], [ [[TMP17]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP20:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true)
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32
-; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[VAL:%.*]], i32 [[TMP15]])
-; IR-ITERATIVE-NEXT: [[TMP17]] = fadd double [[ACCUMULATOR]], [[TMP16]]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = shl i64 1, [[TMP14]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = xor i64 [[TMP18]], -1
-; IR-ITERATIVE-NEXT: [[TMP20]] = and i64 [[ACTIVEBITS]], [[TMP19]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = icmp eq i64 [[TMP20]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP21]], label [[COMPUTEEND:%.*]], label [[COMPUTELOOP]]
-; IR-ITERATIVE: ComputeEnd:
-; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP22]], label [[TMP10:%.*]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_div_value_scope_agent_scope_unsafe(
; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
@@ -897,29 +757,15 @@ define amdgpu_ps void @global_atomic_fadd_double_uni_address_div_value_scope_age
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
-; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.set.inactive.f64(double [[VAL:%.*]], double -0.000000e+00)
-; IR-DPP-NEXT: [[TMP10:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP9]], i32 273, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP11:%.*]] = fadd double [[TMP9]], [[TMP10]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP11]], i32 274, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP13:%.*]] = fadd double [[TMP11]], [[TMP12]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP13]], i32 276, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP15:%.*]] = fadd double [[TMP13]], [[TMP14]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP15]], i32 280, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP17:%.*]] = fadd double [[TMP15]], [[TMP16]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP17]], i32 322, i32 10, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP19:%.*]] = fadd double [[TMP17]], [[TMP18]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP19]], i32 323, i32 12, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP21:%.*]] = fadd double [[TMP19]], [[TMP20]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[TMP21]], i32 63)
-; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP22]])
+; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 2)
; IR-DPP-NEXT: [[TMP24:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP24]], label [[TMP25:%.*]], label [[TMP27:%.*]]
-; IR-DPP: 25:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP26:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP23]] syncscope("agent") monotonic, align 4
; IR-DPP-NEXT: br label [[TMP27]]
-; IR-DPP: 27:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP28]]
-; IR-DPP: 28:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic, align 4
@@ -928,51 +774,45 @@ define amdgpu_ps void @global_atomic_fadd_double_uni_address_div_value_scope_age
define amdgpu_ps void @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #1 {
; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 1) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-ITERATIVE: 14:
+; IR-ITERATIVE: 11:
; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("one-as") monotonic, align 8
; IR-ITERATIVE-NEXT: br label [[TMP16]]
-; IR-ITERATIVE: 16:
+; IR-ITERATIVE: 13:
; IR-ITERATIVE-NEXT: br label [[TMP17]]
-; IR-ITERATIVE: 17:
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
;
; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-DPP: 14:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("one-as") monotonic, align 8
; IR-DPP-NEXT: br label [[TMP16]]
-; IR-DPP: 16:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP17]]
-; IR-DPP: 17:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("one-as") monotonic
@@ -981,73 +821,45 @@ define amdgpu_ps void @global_atomic_fadd_double_uni_address_uni_value_one_as_sc
define amdgpu_ps void @global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, double %val) #1 {
; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
-; IR-ITERATIVE: 10:
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP17:%.*]] syncscope("one-as") monotonic, align 8
-; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
-; IR-ITERATIVE: 12:
-; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 1) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP14:%.*]], label [[TMP12:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP17]] syncscope("one-as") monotonic, align 8
+; IR-ITERATIVE-NEXT: br label [[TMP12]]
; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
-; IR-ITERATIVE: ComputeLoop:
-; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi double [ -0.000000e+00, [[TMP2]] ], [ [[TMP17]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP20:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32
-; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[VAL:%.*]], i32 [[TMP15]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP17]] = call double @llvm.experimental.constrained.fadd.f64(double [[ACCUMULATOR]], double [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = shl i64 1, [[TMP14]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = xor i64 [[TMP18]], -1
-; IR-ITERATIVE-NEXT: [[TMP20]] = and i64 [[ACTIVEBITS]], [[TMP19]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = icmp eq i64 [[TMP20]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP21]], label [[COMPUTEEND:%.*]], label [[COMPUTELOOP]]
-; IR-ITERATIVE: ComputeEnd:
-; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP22]], label [[TMP10:%.*]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP28:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.set.inactive.f64(double [[VAL:%.*]], double -0.000000e+00) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP9]], double [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP13:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP11]], double [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP13]], double [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP15]], double [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP17]], double [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP19]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[TMP21]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP22]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP24:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP24]], label [[TMP25:%.*]], label [[TMP27:%.*]]
-; IR-DPP: 25:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP26:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP23]] syncscope("one-as") monotonic, align 8
; IR-DPP-NEXT: br label [[TMP27]]
-; IR-DPP: 27:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP28]]
-; IR-DPP: 28:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("one-as") monotonic
@@ -1056,51 +868,45 @@ define amdgpu_ps void @global_atomic_fadd_double_uni_address_div_value_one_as_sc
define amdgpu_ps void @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #2 {
; IR-ITERATIVE-LABEL: @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 1) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-ITERATIVE: 14:
+; IR-ITERATIVE: 11:
; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 8
; IR-ITERATIVE-NEXT: br label [[TMP16]]
-; IR-ITERATIVE: 16:
+; IR-ITERATIVE: 13:
; IR-ITERATIVE-NEXT: br label [[TMP17]]
-; IR-ITERATIVE: 17:
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
;
; IR-DPP-LABEL: @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-DPP: 14:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 8
; IR-DPP-NEXT: br label [[TMP16]]
-; IR-DPP: 16:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP17]]
-; IR-DPP: 17:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic
@@ -1109,73 +915,45 @@ define amdgpu_ps void @global_atomic_fsub_double_uni_address_uni_value_agent_sco
define amdgpu_ps void @global_atomic_fsub_double_uni_address_div_value_agent_scope_strictfp(ptr addrspace(1) inreg %ptr, double %val) #2 {
; IR-ITERATIVE-LABEL: @global_atomic_fsub_double_uni_address_div_value_agent_scope_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
-; IR-ITERATIVE: 10:
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], double [[TMP17:%.*]] syncscope("agent") monotonic, align 8
-; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
-; IR-ITERATIVE: 12:
-; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 1) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP14:%.*]], label [[TMP12:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], double [[TMP17]] syncscope("agent") monotonic, align 8
+; IR-ITERATIVE-NEXT: br label [[TMP12]]
; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
-; IR-ITERATIVE: ComputeLoop:
-; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi double [ -0.000000e+00, [[TMP2]] ], [ [[TMP17]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP20:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32
-; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[VAL:%.*]], i32 [[TMP15]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP17]] = call double @llvm.experimental.constrained.fadd.f64(double [[ACCUMULATOR]], double [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = shl i64 1, [[TMP14]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = xor i64 [[TMP18]], -1
-; IR-ITERATIVE-NEXT: [[TMP20]] = and i64 [[ACTIVEBITS]], [[TMP19]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = icmp eq i64 [[TMP20]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP21]], label [[COMPUTEEND:%.*]], label [[COMPUTELOOP]]
-; IR-ITERATIVE: ComputeEnd:
-; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP22]], label [[TMP10:%.*]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fsub_double_uni_address_div_value_agent_scope_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP28:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.set.inactive.f64(double [[VAL:%.*]], double -0.000000e+00) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP9]], double [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP13:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP11]], double [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP13]], double [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP15]], double [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP17]], double [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP19]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[TMP21]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP22]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP24:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP24]], label [[TMP25:%.*]], label [[TMP27:%.*]]
-; IR-DPP: 25:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP26:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], double [[TMP23]] syncscope("agent") monotonic, align 8
; IR-DPP-NEXT: br label [[TMP27]]
-; IR-DPP: 27:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP28]]
-; IR-DPP: 28:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fsub ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic
@@ -1183,25 +961,47 @@ define amdgpu_ps void @global_atomic_fsub_double_uni_address_div_value_agent_sco
}
define amdgpu_ps void @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double inreg %val) #0 {
-; IR-LABEL: @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe(
-; IR-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
-; IR-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]]
-; IR: 2:
-; IR-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
-; IR-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
-; IR-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
-; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
-; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
-; IR-NEXT: [[TMP11:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8
-; IR-NEXT: br label [[TMP12]]
-; IR: 12:
-; IR-NEXT: br label [[TMP13]]
-; IR: 13:
-; IR-NEXT: ret void
+; IR-ITERATIVE-LABEL: @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe(
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
+; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP14:%.*]]
+; IR-ITERATIVE: 2:
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.wave.reduce.fmin.f64(double [[VAL:%.*]], i32 1)
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[TMP9]] syncscope("agent") monotonic, align 8
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP14]]
+; IR-ITERATIVE: 14:
+; IR-ITERATIVE-NEXT: ret void
+;
+; IR-DPP-LABEL: @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe(
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
+; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP14:%.*]]
+; IR-DPP: 2:
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
+; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.wave.reduce.fmin.f64(double [[VAL:%.*]], i32 2)
+; IR-DPP-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-DPP-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]
+; IR-DPP: 11:
+; IR-DPP-NEXT: [[TMP12:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[TMP9]] syncscope("agent") monotonic, align 8
+; IR-DPP-NEXT: br label [[TMP13]]
+; IR-DPP: 13:
+; IR-DPP-NEXT: br label [[TMP14]]
+; IR-DPP: 14:
+; IR-DPP-NEXT: ret void
;
%result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic
ret void
@@ -1218,30 +1018,16 @@ define amdgpu_ps void @global_atomic_fmin_double_uni_address_div_value_agent_sco
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
-; IR-ITERATIVE: 10:
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[TMP17:%.*]] syncscope("agent") monotonic, align 8
-; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
-; IR-ITERATIVE: 12:
-; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = call double @llvm.amdgcn.wave.reduce.fmin.f64(double [[VAL:%.*]], i32 1)
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP14:%.*]], label [[TMP12:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[TMP17]] syncscope("agent") monotonic, align 8
+; IR-ITERATIVE-NEXT: br label [[TMP12]]
; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
-; IR-ITERATIVE: ComputeLoop:
-; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi double [ +qnan, [[TMP2]] ], [ [[TMP17]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP20:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true)
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32
-; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[VAL:%.*]], i32 [[TMP15]])
-; IR-ITERATIVE-NEXT: [[TMP17]] = call double @llvm.minnum.f64(double [[ACCUMULATOR]], double [[TMP16]])
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = shl i64 1, [[TMP14]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = xor i64 [[TMP18]], -1
-; IR-ITERATIVE-NEXT: [[TMP20]] = and i64 [[ACTIVEBITS]], [[TMP19]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = icmp eq i64 [[TMP20]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP21]], label [[COMPUTEEND:%.*]], label [[COMPUTELOOP]]
-; IR-ITERATIVE: ComputeEnd:
-; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP22]], label [[TMP10:%.*]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe(
; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
@@ -1253,29 +1039,15 @@ define amdgpu_ps void @global_atomic_fmin_double_uni_address_div_value_agent_sco
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
-; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.set.inactive.f64(double [[VAL:%.*]], double +qnan)
-; IR-DPP-NEXT: [[TMP10:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP9]], i32 273, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.minnum.f64(double [[TMP9]], double [[TMP10]])
-; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP11]], i32 274, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP13:%.*]] = call double @llvm.minnum.f64(double [[TMP11]], double [[TMP12]])
-; IR-DPP-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP13]], i32 276, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP15:%.*]] = call double @llvm.minnum.f64(double [[TMP13]], double [[TMP14]])
-; IR-DPP-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP15]], i32 280, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP17:%.*]] = call double @llvm.minnum.f64(double [[TMP15]], double [[TMP16]])
-; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP17]], i32 322, i32 10, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.minnum.f64(double [[TMP17]], double [[TMP18]])
-; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP19]], i32 323, i32 12, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.minnum.f64(double [[TMP19]], double [[TMP20]])
-; IR-DPP-NEXT: [[TMP22:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[TMP21]], i32 63)
-; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP22]])
+; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.wave.reduce.fmin.f64(double [[VAL:%.*]], i32 2)
; IR-DPP-NEXT: [[TMP24:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP24]], label [[TMP25:%.*]], label [[TMP27:%.*]]
-; IR-DPP: 25:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP26:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[TMP23]] syncscope("agent") monotonic, align 8
; IR-DPP-NEXT: br label [[TMP27]]
-; IR-DPP: 27:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP28]]
-; IR-DPP: 28:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic
@@ -1284,43 +1056,45 @@ define amdgpu_ps void @global_atomic_fmin_double_uni_address_div_value_agent_sco
define amdgpu_ps void @global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #1{
; IR-ITERATIVE-LABEL: @global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[VAL:%.*]] = call double @llvm.amdgcn.wave.reduce.fmax.f64(double [[VAL1:%.*]], i32 1) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR-ITERATIVE: 10:
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL]] syncscope("agent") monotonic, align 8
; IR-ITERATIVE-NEXT: br label [[TMP12]]
-; IR-ITERATIVE: 12:
-; IR-ITERATIVE-NEXT: br label [[TMP13]]
; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
;
; IR-DPP-LABEL: @global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[VAL:%.*]] = call double @llvm.amdgcn.wave.reduce.fmax.f64(double [[VAL1:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR-DPP: 10:
-; IR-DPP-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8
+; IR-DPP: 11:
+; IR-DPP-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL]] syncscope("agent") monotonic, align 8
; IR-DPP-NEXT: br label [[TMP12]]
-; IR-DPP: 12:
-; IR-DPP-NEXT: br label [[TMP13]]
; IR-DPP: 13:
+; IR-DPP-NEXT: br label [[TMP13]]
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic
@@ -1329,73 +1103,45 @@ define amdgpu_ps void @global_atomic_fmax_double_uni_address_uni_value_agent_sco
define amdgpu_ps void @global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, double %val) #1{
; IR-ITERATIVE-LABEL: @global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
-; IR-ITERATIVE: 10:
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[TMP17:%.*]] syncscope("agent") monotonic, align 8
-; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
-; IR-ITERATIVE: 12:
-; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = call double @llvm.amdgcn.wave.reduce.fmax.f64(double [[VAL:%.*]], i32 1) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP14:%.*]], label [[TMP12:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[TMP17]] syncscope("agent") monotonic, align 8
+; IR-ITERATIVE-NEXT: br label [[TMP12]]
; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
-; IR-ITERATIVE: ComputeLoop:
-; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi double [ +qnan, [[TMP2]] ], [ [[TMP17]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP20:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32
-; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[VAL:%.*]], i32 [[TMP15]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP17]] = call double @llvm.experimental.constrained.maxnum.f64(double [[ACCUMULATOR]], double [[TMP16]], metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = shl i64 1, [[TMP14]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = xor i64 [[TMP18]], -1
-; IR-ITERATIVE-NEXT: [[TMP20]] = and i64 [[ACTIVEBITS]], [[TMP19]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = icmp eq i64 [[TMP20]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP21]], label [[COMPUTEEND:%.*]], label [[COMPUTELOOP]]
-; IR-ITERATIVE: ComputeEnd:
-; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP22]], label [[TMP10:%.*]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP28:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.set.inactive.f64(double [[VAL:%.*]], double +qnan) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP9]], double [[TMP10]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP13:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP11]], double [[TMP12]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP13]], double [[TMP14]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP15]], double [[TMP16]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP17]], double [[TMP18]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP19]], double [[TMP20]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[TMP21]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP22]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.wave.reduce.fmax.f64(double [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP24:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP24]], label [[TMP25:%.*]], label [[TMP27:%.*]]
-; IR-DPP: 25:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP26:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[TMP23]] syncscope("agent") monotonic, align 8
; IR-DPP-NEXT: br label [[TMP27]]
-; IR-DPP: 27:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP28]]
-; IR-DPP: 28:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic
@@ -1404,51 +1150,45 @@ define amdgpu_ps void @global_atomic_fmax_double_uni_address_div_value_agent_sco
define amdgpu_ps void @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #2 {
; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 1) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-ITERATIVE: 14:
+; IR-ITERATIVE: 11:
; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] monotonic, align 4
; IR-ITERATIVE-NEXT: br label [[TMP16]]
-; IR-ITERATIVE: 16:
+; IR-ITERATIVE: 13:
; IR-ITERATIVE-NEXT: br label [[TMP17]]
-; IR-ITERATIVE: 17:
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
;
; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-DPP: 14:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] monotonic, align 4
; IR-DPP-NEXT: br label [[TMP16]]
-; IR-DPP: 16:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP17]]
-; IR-DPP: 17:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, double %val monotonic, align 4
@@ -1457,73 +1197,45 @@ define amdgpu_ps void @global_atomic_fadd_double_uni_address_uni_value_system_sc
define amdgpu_ps void @global_atomic_fadd_double_uni_address_div_value_system_scope_strictfp(ptr addrspace(1) inreg %ptr, double %val) #2 {
; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_div_value_system_scope_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
-; IR-ITERATIVE: 10:
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP17:%.*]] monotonic, align 4
-; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
-; IR-ITERATIVE: 12:
-; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 1) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP14:%.*]], label [[TMP12:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP17]] monotonic, align 4
+; IR-ITERATIVE-NEXT: br label [[TMP12]]
; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
-; IR-ITERATIVE: ComputeLoop:
-; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi double [ -0.000000e+00, [[TMP2]] ], [ [[TMP17]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP20:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32
-; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[VAL:%.*]], i32 [[TMP15]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP17]] = call double @llvm.experimental.constrained.fadd.f64(double [[ACCUMULATOR]], double [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = shl i64 1, [[TMP14]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = xor i64 [[TMP18]], -1
-; IR-ITERATIVE-NEXT: [[TMP20]] = and i64 [[ACTIVEBITS]], [[TMP19]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = icmp eq i64 [[TMP20]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP21]], label [[COMPUTEEND:%.*]], label [[COMPUTELOOP]]
-; IR-ITERATIVE: ComputeEnd:
-; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP22]], label [[TMP10:%.*]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_div_value_system_scope_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP28:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.set.inactive.f64(double [[VAL:%.*]], double -0.000000e+00) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP9]], double [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP13:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP11]], double [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP13]], double [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP15]], double [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP17]], double [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP19]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[TMP21]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP22]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP24:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP24]], label [[TMP25:%.*]], label [[TMP27:%.*]]
-; IR-DPP: 25:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP26:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP23]] monotonic, align 4
; IR-DPP-NEXT: br label [[TMP27]]
-; IR-DPP: 27:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP28]]
-; IR-DPP: 28:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, double %val monotonic, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
index d307ffaff5c63..6a1f43b5582e0 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
@@ -18,35 +18,36 @@ declare double @div.double.value()
define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 {
; GFX7LESS-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB0_3
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX7LESS-NEXT: v_mul_f32_e32 v2, 4.0, v0
-; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s2
+; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_add_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB0_2
; GFX7LESS-NEXT: .LBB0_3:
@@ -54,27 +55,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
;
; GFX9-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s6, v0
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB0_3
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
-; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s4
; GFX9-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -86,27 +88,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
;
; GFX1064-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe:
; GFX1064: ; %bb.0:
-; GFX1064-NEXT: s_mov_b64 s[2:3], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-NEXT: v_readfirstlane_b32 s6, v0
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB0_3
; GFX1064-NEXT: ; %bb.1:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
-; GFX1064-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX1064-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v1, s4
+; GFX1064-NEXT: v_mov_b32_e32 v1, s2
+; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -118,31 +121,32 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
;
; GFX1032-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: s_mov_b32 s3, exec_lo
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s3, 0
+; GFX1032-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB0_3
; GFX1032-NEXT: ; %bb.1:
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
-; GFX1032-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB0_2
; GFX1032-NEXT: .LBB0_3:
; GFX1032-NEXT: s_endpgm
@@ -150,74 +154,78 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
; GFX1164-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe:
; GFX1164: ; %bb.0:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
; GFX1164-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1164-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB0_2
; GFX1164-NEXT: ; %bb.1:
; GFX1164-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164-NEXT: v_cvt_f32_ubyte0_e32 v0, s0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-NEXT: v_mov_b32_e32 v1, s0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_atomic_add_f32 v1, v0, s[2:3]
+; GFX1164-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GFX1164-NEXT: .LBB0_2:
; GFX1164-NEXT: s_endpgm
;
; GFX1132-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe:
; GFX1132: ; %bb.0:
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
; GFX1132-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1132-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB0_2
; GFX1132-NEXT: ; %bb.1:
; GFX1132-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_cvt_f32_ubyte0_e32 v0, s0
-; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mul_f32 v0, 4.0, v0
+; GFX1132-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_atomic_add_f32 v1, v0, s[2:3]
+; GFX1132-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GFX1132-NEXT: .LBB0_2:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB0_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
-; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, s6
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, s2
+; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
; GFX7LESS-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-DPP-NEXT: v_add_f32_e32 v0, s6, v1
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB0_2
; GFX7LESS-DPP-NEXT: .LBB0_3:
@@ -225,27 +233,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
;
; GFX9-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s6, v0
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB0_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
-; GFX9-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX9-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -257,27 +266,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
;
; GFX1064-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe:
; GFX1064-DPP: ; %bb.0:
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s6, v0
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB0_3
; GFX1064-DPP-NEXT: ; %bb.1:
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX1064-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX1064-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s4
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s2
+; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -289,31 +299,32 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
;
; GFX1032-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: s_mov_b32 s3, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
+; GFX1032-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s2, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB0_3
; GFX1032-DPP-NEXT: ; %bb.1:
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
-; GFX1032-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB0_2
; GFX1032-DPP-NEXT: .LBB0_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -321,40 +332,43 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
; GFX1164-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe:
; GFX1164-DPP: ; %bb.0:
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1164-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB0_2
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_atomic_add_f32 v1, v0, s[2:3]
+; GFX1164-DPP-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GFX1164-DPP-NEXT: .LBB0_2:
; GFX1164-DPP-NEXT: s_endpgm
;
; GFX1132-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe:
; GFX1132-DPP: ; %bb.0:
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
; GFX1132-DPP-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1132-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB0_2
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s0
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mul_f32 v0, 4.0, v0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_atomic_add_f32 v1, v0, s[2:3]
+; GFX1132-DPP-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GFX1132-DPP-NEXT: .LBB0_2:
; GFX1132-DPP-NEXT: s_endpgm
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory !1, !amdgpu.ignore.denormal.mode !1
@@ -393,24 +407,22 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX7LESS-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_brev_b32 s6, 1
+; GFX7LESS-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s2
-; GFX7LESS-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f32_e32 v2, s4, v2
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB1_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: v_readlane_b32 s3, v0, s2
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s3
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX7LESS-NEXT: v_add_f32_e32 v2, s6, v2
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v2
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB1_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
@@ -422,15 +434,15 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX7LESS-NEXT: .LBB1_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_add_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB1_4
; GFX7LESS-NEXT: .LBB1_5:
@@ -466,34 +478,34 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX9-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_brev_b32 s4, 1
+; GFX9-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s4, v0, s2
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_readlane_b32 s3, v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: s_bitset0_b64 s[0:1], s2
; GFX9-NEXT: v_add_f32_e32 v2, s4, v2
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9-NEXT: s_cbranch_execz .LBB1_5
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX9-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-NEXT: .LBB1_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_add_f32_e32 v0, s4, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -533,34 +545,33 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_brev_b32 s4, 1
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1064-NEXT: v_add_f32_e32 v2, s3, v2
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX1064-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1064-NEXT: v_add_f32_e64 v2, s4, s3
+; GFX1064-NEXT: v_readfirstlane_b32 s4, v2
; GFX1064-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064-NEXT: s_cbranch_execz .LBB1_5
; GFX1064-NEXT: ; %bb.3:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1064-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-NEXT: .LBB1_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_add_f32_e32 v0, s4, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -600,38 +611,37 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_brev_b32 s2, 1
+; GFX1032-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f32_e32 v2, s2, v2
+; GFX1032-NEXT: v_readlane_b32 s3, v0, s1
+; GFX1032-NEXT: s_bitset0_b32 s0, s1
+; GFX1032-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1032-NEXT: v_add_f32_e64 v2, s2, s3
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v2
; GFX1032-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1032-NEXT: s_cbranch_execz .LBB1_5
; GFX1032-NEXT: ; %bb.3:
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
+; GFX1032-NEXT: s_mov_b32 s3, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1032-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-NEXT: .LBB1_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB1_4
; GFX1032-NEXT: .LBB1_5:
; GFX1032-NEXT: s_endpgm
@@ -656,31 +666,31 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_bfrev_b32_e32 v1, 1
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
-; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f32_e32 v1, s3, v1
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: s_brev_b32 s2, 1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1164-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: v_add_f32_e64 v2, s2, s4
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_readfirstlane_b32 s2, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: ; %bb.2:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB1_4
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-NEXT: v_mov_b32_e32 v0, s2
+; GFX1164-NEXT: v_mov_b32_e32 v1, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
+; GFX1164-NEXT: global_atomic_add_f32 v1, v0, s[0:1]
; GFX1164-NEXT: .LBB1_4:
; GFX1164-NEXT: s_endpgm
;
@@ -705,30 +715,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f32_e32 v1, s2, v1
-; GFX1132-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: s_brev_b32 s0, 1
+; GFX1132-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1132-NEXT: s_bitset0_b32 s1, s2
+; GFX1132-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132-NEXT: v_add_f32_e64 v2, s0, s3
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1132-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB1_4
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v0, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
+; GFX1132-NEXT: global_atomic_add_f32 v1, v0, s[2:3]
; GFX1132-NEXT: .LBB1_4:
; GFX1132-NEXT: s_endpgm
;
@@ -818,30 +826,19 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[0:1]
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v4 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v3, v4, v3
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
@@ -898,44 +895,42 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1064-DPP-NEXT: ds_permute_b32 v4, v5, v3
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v3, 32
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_add_f32_e64 v3, s2, s3
+; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB1_3
; GFX1064-DPP-NEXT: ; %bb.1:
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-DPP-NEXT: .LBB1_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f32_e32 v0, s4, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -976,43 +971,37 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s0
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX1032-DPP-NEXT: v_readlane_b32 s2, v3, 31
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB1_3
; GFX1032-DPP-NEXT: ; %bb.1:
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-DPP-NEXT: .LBB1_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB1_2
; GFX1032-DPP-NEXT: .LBB1_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -1038,46 +1027,44 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s[0:1]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1164-DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v2, v1
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b32 s0, s2
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_atomic_add_f32 v4, v0, s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_add_f32 v4, v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB1_2:
; GFX1164-DPP-NEXT: s_endpgm
;
@@ -1103,39 +1090,30 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v1, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s0, s1
+; GFX1132-DPP-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_atomic_add_f32 v4, v0, s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_add_f32 v3, v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB1_2:
; GFX1132-DPP-NEXT: s_endpgm
%divValue = call float @div.float.value()
@@ -1146,45 +1124,36 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp(ptr addrspace(1) %ptr) #1 {
; GFX7LESS-LABEL: global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT: s_mov_b32 s14, -1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0
-; GFX7LESS-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0
-; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB2_3
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dword s2, s[0:1], 0x0
-; GFX7LESS-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v1, s2
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_add_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB2_2
; GFX7LESS-NEXT: .LBB2_3:
@@ -1192,37 +1161,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_one_as_scope
;
; GFX9-LABEL: global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s14, -1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-NEXT: s_add_u32 s12, s12, s11
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s6, v0
+; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB2_3
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX9-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -1235,34 +1195,27 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_one_as_scope
; GFX1064-LABEL: global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT: s_mov_b32 s14, -1
-; GFX1064-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB2_3
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1064-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -1275,136 +1228,110 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_one_as_scope
; GFX1032-LABEL: global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1032: ; %bb.0:
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT: s_mov_b32 s14, -1
-; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s3, 0
+; GFX1032-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB2_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mov_b32_e32 v1, s3
-; GFX1032-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB2_2
; GFX1032-NEXT: .LBB2_3:
; GFX1032-NEXT: s_endpgm
;
; GFX1164-LABEL: global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1164: ; %bb.0:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_clause 0x1
-; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB2_2
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-NEXT: v_mov_b32_e32 v1, s0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_atomic_add_f32 v1, v0, s[0:1]
+; GFX1164-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GFX1164-NEXT: .LBB2_2:
; GFX1164-NEXT: s_endpgm
;
; GFX1132-LABEL: global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_clause 0x1
-; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB2_2
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
-; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mul_f32 v0, 4.0, v0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_atomic_add_f32 v1, v0, s[0:1]
+; GFX1132-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GFX1132-NEXT: .LBB2_2:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT: s_mov_b32 s14, -1
; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-DPP-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0
-; GFX7LESS-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0
-; GFX7LESS-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB2_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, s2
; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
; GFX7LESS-DPP-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-DPP-NEXT: v_add_f32_e32 v0, s6, v1
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB2_2
; GFX7LESS-DPP-NEXT: .LBB2_3:
@@ -1412,37 +1339,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_one_as_scope
;
; GFX9-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT: s_mov_b32 s14, -1
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-DPP-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB2_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX9-DPP-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -1455,34 +1373,27 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_one_as_scope
; GFX1064-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1064-DPP: ; %bb.0:
; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1064-DPP-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB2_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1064-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -1495,92 +1406,75 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_one_as_scope
; GFX1032-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1032-DPP: ; %bb.0:
; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
+; GFX1032-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB2_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s3
-; GFX1032-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-DPP-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB2_2
; GFX1032-DPP-NEXT: .LBB2_3:
; GFX1032-DPP-NEXT: s_endpgm
;
; GFX1164-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1164-DPP: ; %bb.0:
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_clause 0x1
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB2_2
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_atomic_add_f32 v1, v0, s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GFX1164-DPP-NEXT: .LBB2_2:
; GFX1164-DPP-NEXT: s_endpgm
;
; GFX1132-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_clause 0x1
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB2_2
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mul_f32 v0, 4.0, v0
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_atomic_add_f32 v1, v0, s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GFX1132-DPP-NEXT: .LBB2_2:
; GFX1132-DPP-NEXT: s_endpgm
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("one-as") monotonic, !amdgpu.no.fine.grained.memory !1, !amdgpu.ignore.denormal.mode !1
@@ -1620,24 +1514,22 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX7LESS-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_brev_b32 s6, 1
+; GFX7LESS-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s2
-; GFX7LESS-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f32_e32 v2, s4, v2
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB3_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: v_readlane_b32 s3, v0, s2
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s3
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX7LESS-NEXT: v_add_f32_e32 v2, s6, v2
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v2
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB3_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB3_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
@@ -1649,15 +1541,15 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX7LESS-NEXT: .LBB3_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_add_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB3_4
; GFX7LESS-NEXT: .LBB3_5:
@@ -1693,34 +1585,34 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX9-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_brev_b32 s4, 1
+; GFX9-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s4, v0, s2
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_readlane_b32 s3, v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: s_bitset0_b64 s[0:1], s2
; GFX9-NEXT: v_add_f32_e32 v2, s4, v2
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9-NEXT: s_cbranch_execz .LBB3_5
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX9-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-NEXT: .LBB3_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_add_f32_e32 v0, s4, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -1760,34 +1652,33 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_brev_b32 s4, 1
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1064-NEXT: v_add_f32_e32 v2, s3, v2
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX1064-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1064-NEXT: v_add_f32_e64 v2, s4, s3
+; GFX1064-NEXT: v_readfirstlane_b32 s4, v2
; GFX1064-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064-NEXT: s_cbranch_execz .LBB3_5
; GFX1064-NEXT: ; %bb.3:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1064-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-NEXT: .LBB3_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_add_f32_e32 v0, s4, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -1827,38 +1718,37 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_brev_b32 s2, 1
+; GFX1032-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f32_e32 v2, s2, v2
+; GFX1032-NEXT: v_readlane_b32 s3, v0, s1
+; GFX1032-NEXT: s_bitset0_b32 s0, s1
+; GFX1032-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1032-NEXT: v_add_f32_e64 v2, s2, s3
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v2
; GFX1032-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1032-NEXT: s_cbranch_execz .LBB3_5
; GFX1032-NEXT: ; %bb.3:
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
+; GFX1032-NEXT: s_mov_b32 s3, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1032-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-NEXT: .LBB3_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB3_4
; GFX1032-NEXT: .LBB3_5:
; GFX1032-NEXT: s_endpgm
@@ -1883,31 +1773,31 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_bfrev_b32_e32 v1, 1
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
-; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f32_e32 v1, s3, v1
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: s_brev_b32 s2, 1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1164-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: v_add_f32_e64 v2, s2, s4
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_readfirstlane_b32 s2, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: ; %bb.2:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB3_4
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-NEXT: v_mov_b32_e32 v0, s2
+; GFX1164-NEXT: v_mov_b32_e32 v1, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
+; GFX1164-NEXT: global_atomic_add_f32 v1, v0, s[0:1]
; GFX1164-NEXT: .LBB3_4:
; GFX1164-NEXT: s_endpgm
;
@@ -1932,30 +1822,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f32_e32 v1, s2, v1
-; GFX1132-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: s_brev_b32 s0, 1
+; GFX1132-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1132-NEXT: s_bitset0_b32 s1, s2
+; GFX1132-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132-NEXT: v_add_f32_e64 v2, s0, s3
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1132-NEXT: s_cbranch_scc1 .LBB3_1
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB3_4
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v0, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
+; GFX1132-NEXT: global_atomic_add_f32 v1, v0, s[2:3]
; GFX1132-NEXT: .LBB3_4:
; GFX1132-NEXT: s_endpgm
;
@@ -2045,30 +1933,19 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[0:1]
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v4 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v3, v4, v3
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
@@ -2125,44 +2002,42 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1064-DPP-NEXT: ds_permute_b32 v4, v5, v3
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v3, 32
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_add_f32_e64 v3, s2, s3
+; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB3_3
; GFX1064-DPP-NEXT: ; %bb.1:
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-DPP-NEXT: .LBB3_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f32_e32 v0, s4, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -2203,43 +2078,37 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s0
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX1032-DPP-NEXT: v_readlane_b32 s2, v3, 31
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB3_3
; GFX1032-DPP-NEXT: ; %bb.1:
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-DPP-NEXT: .LBB3_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB3_2
; GFX1032-DPP-NEXT: .LBB3_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -2265,46 +2134,44 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s[0:1]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1164-DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v2, v1
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b32 s0, s2
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_atomic_add_f32 v4, v0, s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_add_f32 v4, v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB3_2:
; GFX1164-DPP-NEXT: s_endpgm
;
@@ -2330,39 +2197,30 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v1, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s0, s1
+; GFX1132-DPP-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_atomic_add_f32 v4, v0, s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_add_f32 v3, v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB3_2:
; GFX1132-DPP-NEXT: s_endpgm
%divValue = call float @div.float.value() strictfp
@@ -2373,45 +2231,36 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp(ptr addrspace(1) %ptr) #2{
; GFX7LESS-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT: s_mov_b32 s14, -1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0
-; GFX7LESS-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0
-; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB4_3
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dword s2, s[0:1], 0x0
-; GFX7LESS-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v1, s2
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_add_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB4_2
; GFX7LESS-NEXT: .LBB4_3:
@@ -2419,37 +2268,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
;
; GFX9-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s14, -1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-NEXT: s_add_u32 s12, s12, s11
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s6, v0
+; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB4_3
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX9-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -2462,34 +2302,27 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
; GFX1064-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp:
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT: s_mov_b32 s14, -1
-; GFX1064-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB4_3
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1064-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -2502,75 +2335,63 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
; GFX1032-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp:
; GFX1032: ; %bb.0:
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT: s_mov_b32 s14, -1
-; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s3, 0
+; GFX1032-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB4_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mov_b32_e32 v1, s3
-; GFX1032-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB4_2
; GFX1032-NEXT: .LBB4_3:
; GFX1032-NEXT: s_endpgm
;
; GFX1164-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp:
; GFX1164: ; %bb.0:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_clause 0x1
-; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB4_3
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v3, 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_load_b32 s2, s[0:1], 0x0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX1164-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: v_mov_b32_e32 v1, s2
; GFX1164-NEXT: s_mov_b64 s[2:3], 0
-; GFX1164-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1164-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-NEXT: v_mov_b32_e32 v1, v0
@@ -2583,85 +2404,71 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
;
; GFX1132-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_clause 0x1
-; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-NEXT: s_mov_b32 s3, 0
+; GFX1132-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB4_3
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v3, 0
+; GFX1132-NEXT: v_mov_b32_e32 v2, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b32 s3, s[0:1], 0x0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1132-NEXT: s_load_b32 s4, s[0:1], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mul_f32 v2, 4.0, v0
+; GFX1132-NEXT: v_mov_b32_e32 v1, s4
; GFX1132-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-NEXT: s_cbranch_execnz .LBB4_2
; GFX1132-NEXT: .LBB4_3:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT: s_mov_b32 s14, -1
; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-DPP-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0
-; GFX7LESS-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0
-; GFX7LESS-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB4_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, s2
; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
; GFX7LESS-DPP-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-DPP-NEXT: v_add_f32_e32 v0, s6, v1
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB4_2
; GFX7LESS-DPP-NEXT: .LBB4_3:
@@ -2669,37 +2476,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
;
; GFX9-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT: s_mov_b32 s14, -1
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-DPP-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB4_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX9-DPP-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -2712,34 +2510,27 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
; GFX1064-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp:
; GFX1064-DPP: ; %bb.0:
; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1064-DPP-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB4_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1064-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -2752,75 +2543,63 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
; GFX1032-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp:
; GFX1032-DPP: ; %bb.0:
; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
+; GFX1032-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB4_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s3
-; GFX1032-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-DPP-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB4_2
; GFX1032-DPP-NEXT: .LBB4_3:
; GFX1032-DPP-NEXT: s_endpgm
;
; GFX1164-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp:
; GFX1164-DPP: ; %bb.0:
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_clause 0x1
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB4_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_load_b32 s2, s[0:1], 0x0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s2
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX1164-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1164-DPP-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -2833,40 +2612,35 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
;
; GFX1132-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_clause 0x1
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s3, 0
+; GFX1132-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB4_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_load_b32 s3, s[0:1], 0x0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1132-DPP-NEXT: s_load_b32 s4, s[0:1], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mul_f32 v2, 4.0, v0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1132-DPP-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB4_2
; GFX1132-DPP-NEXT: .LBB4_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -2907,24 +2681,22 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX7LESS-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_brev_b32 s6, 1
+; GFX7LESS-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s2
-; GFX7LESS-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f32_e32 v2, s4, v2
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB5_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: v_readlane_b32 s3, v0, s2
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s3
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX7LESS-NEXT: v_add_f32_e32 v2, s6, v2
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v2
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB5_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB5_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
@@ -2936,15 +2708,15 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX7LESS-NEXT: .LBB5_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_add_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB5_4
; GFX7LESS-NEXT: .LBB5_5:
@@ -2980,34 +2752,34 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX9-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_brev_b32 s4, 1
+; GFX9-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s4, v0, s2
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_readlane_b32 s3, v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: s_bitset0_b64 s[0:1], s2
; GFX9-NEXT: v_add_f32_e32 v2, s4, v2
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9-NEXT: s_cbranch_execz .LBB5_5
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX9-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-NEXT: .LBB5_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_add_f32_e32 v0, s4, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -3047,34 +2819,33 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_brev_b32 s4, 1
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1064-NEXT: v_add_f32_e32 v2, s3, v2
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX1064-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1064-NEXT: v_add_f32_e64 v2, s4, s3
+; GFX1064-NEXT: v_readfirstlane_b32 s4, v2
; GFX1064-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064-NEXT: s_cbranch_execz .LBB5_5
; GFX1064-NEXT: ; %bb.3:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1064-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-NEXT: .LBB5_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_add_f32_e32 v0, s4, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -3114,38 +2885,37 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_brev_b32 s2, 1
+; GFX1032-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f32_e32 v2, s2, v2
+; GFX1032-NEXT: v_readlane_b32 s3, v0, s1
+; GFX1032-NEXT: s_bitset0_b32 s0, s1
+; GFX1032-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1032-NEXT: v_add_f32_e64 v2, s2, s3
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v2
; GFX1032-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1032-NEXT: s_cbranch_execz .LBB5_5
; GFX1032-NEXT: ; %bb.3:
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
+; GFX1032-NEXT: s_mov_b32 s3, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1032-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-NEXT: .LBB5_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB5_4
; GFX1032-NEXT: .LBB5_5:
; GFX1032-NEXT: s_endpgm
@@ -3170,31 +2940,31 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_bfrev_b32_e32 v1, 1
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
-; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f32_e32 v1, s3, v1
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: s_brev_b32 s2, 1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1164-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: v_add_f32_e64 v2, s2, s4
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_readfirstlane_b32 s2, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: ; %bb.2:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB5_4
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-NEXT: v_mov_b32_e32 v0, s2
+; GFX1164-NEXT: v_mov_b32_e32 v1, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
+; GFX1164-NEXT: global_atomic_add_f32 v1, v0, s[0:1]
; GFX1164-NEXT: .LBB5_4:
; GFX1164-NEXT: s_endpgm
;
@@ -3219,30 +2989,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f32_e32 v1, s2, v1
-; GFX1132-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: s_brev_b32 s0, 1
+; GFX1132-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1132-NEXT: s_bitset0_b32 s1, s2
+; GFX1132-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132-NEXT: v_add_f32_e64 v2, s0, s3
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1132-NEXT: s_cbranch_scc1 .LBB5_1
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB5_4
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v0, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
+; GFX1132-NEXT: global_atomic_add_f32 v1, v0, s[2:3]
; GFX1132-NEXT: .LBB5_4:
; GFX1132-NEXT: s_endpgm
;
@@ -3332,30 +3100,19 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[0:1]
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v4 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v3, v4, v3
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
@@ -3412,44 +3169,42 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1064-DPP-NEXT: ds_permute_b32 v4, v5, v3
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v3, 32
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_add_f32_e64 v3, s2, s3
+; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB5_3
; GFX1064-DPP-NEXT: ; %bb.1:
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-DPP-NEXT: .LBB5_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f32_e32 v0, s4, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -3490,43 +3245,37 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s0
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX1032-DPP-NEXT: v_readlane_b32 s2, v3, 31
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB5_3
; GFX1032-DPP-NEXT: ; %bb.1:
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-DPP-NEXT: .LBB5_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB5_2
; GFX1032-DPP-NEXT: .LBB5_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -3552,46 +3301,44 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s[0:1]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1164-DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v2, v1
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b32 s0, s2
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_atomic_add_f32 v4, v0, s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_add_f32 v4, v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB5_2:
; GFX1164-DPP-NEXT: s_endpgm
;
@@ -3617,39 +3364,30 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v1, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s0, s1
+; GFX1132-DPP-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_atomic_add_f32 v4, v0, s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_add_f32 v3, v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB5_2:
; GFX1132-DPP-NEXT: s_endpgm
%divValue = call float @div.float.value()
@@ -3690,24 +3428,22 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX7LESS-NEXT: .LBB6_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_brev_b32 s6, 1
+; GFX7LESS-NEXT: .LBB6_1: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s2
-; GFX7LESS-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f32_e32 v2, s4, v2
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB6_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: v_readlane_b32 s3, v0, s2
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s3
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX7LESS-NEXT: v_add_f32_e32 v2, s6, v2
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v2
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB6_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB6_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
@@ -3719,15 +3455,15 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX7LESS-NEXT: .LBB6_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_add_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB6_4
; GFX7LESS-NEXT: .LBB6_5:
@@ -3763,34 +3499,34 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX9-NEXT: .LBB6_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_brev_b32 s4, 1
+; GFX9-NEXT: .LBB6_1: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s4, v0, s2
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_readlane_b32 s3, v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: s_bitset0_b64 s[0:1], s2
; GFX9-NEXT: v_add_f32_e32 v2, s4, v2
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: s_cbranch_scc1 .LBB6_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9-NEXT: s_cbranch_execz .LBB6_5
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX9-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-NEXT: .LBB6_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_add_f32_e32 v0, s4, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -3830,34 +3566,33 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB6_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_brev_b32 s4, 1
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: .LBB6_1: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1064-NEXT: v_add_f32_e32 v2, s3, v2
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX1064-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1064-NEXT: v_add_f32_e64 v2, s4, s3
+; GFX1064-NEXT: v_readfirstlane_b32 s4, v2
; GFX1064-NEXT: s_cbranch_scc1 .LBB6_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064-NEXT: s_cbranch_execz .LBB6_5
; GFX1064-NEXT: ; %bb.3:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1064-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-NEXT: .LBB6_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_add_f32_e32 v0, s4, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -3897,38 +3632,37 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB6_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_brev_b32 s2, 1
+; GFX1032-NEXT: .LBB6_1: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f32_e32 v2, s2, v2
+; GFX1032-NEXT: v_readlane_b32 s3, v0, s1
+; GFX1032-NEXT: s_bitset0_b32 s0, s1
+; GFX1032-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1032-NEXT: v_add_f32_e64 v2, s2, s3
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v2
; GFX1032-NEXT: s_cbranch_scc1 .LBB6_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1032-NEXT: s_cbranch_execz .LBB6_5
; GFX1032-NEXT: ; %bb.3:
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
+; GFX1032-NEXT: s_mov_b32 s3, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1032-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-NEXT: .LBB6_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB6_4
; GFX1032-NEXT: .LBB6_5:
; GFX1032-NEXT: s_endpgm
@@ -3953,31 +3687,31 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_bfrev_b32_e32 v1, 1
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB6_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
-; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f32_e32 v1, s3, v1
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: s_brev_b32 s2, 1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: .LBB6_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1164-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: v_add_f32_e64 v2, s2, s4
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_readfirstlane_b32 s2, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB6_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: ; %bb.2:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB6_4
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-NEXT: v_mov_b32_e32 v0, s2
+; GFX1164-NEXT: v_mov_b32_e32 v1, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
+; GFX1164-NEXT: global_atomic_add_f32 v1, v0, s[0:1]
; GFX1164-NEXT: .LBB6_4:
; GFX1164-NEXT: s_endpgm
;
@@ -4002,30 +3736,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB6_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f32_e32 v1, s2, v1
-; GFX1132-NEXT: s_cbranch_scc1 .LBB6_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: s_brev_b32 s0, 1
+; GFX1132-NEXT: .LBB6_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1132-NEXT: s_bitset0_b32 s1, s2
+; GFX1132-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132-NEXT: v_add_f32_e64 v2, s0, s3
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1132-NEXT: s_cbranch_scc1 .LBB6_1
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB6_4
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v0, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
+; GFX1132-NEXT: global_atomic_add_f32 v1, v0, s[2:3]
; GFX1132-NEXT: .LBB6_4:
; GFX1132-NEXT: s_endpgm
;
@@ -4115,30 +3847,19 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[0:1]
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v4 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v3, v4, v3
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
@@ -4195,44 +3916,42 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1064-DPP-NEXT: ds_permute_b32 v4, v5, v3
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v3, 32
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_add_f32_e64 v3, s2, s3
+; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB6_3
; GFX1064-DPP-NEXT: ; %bb.1:
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-DPP-NEXT: .LBB6_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f32_e32 v0, s4, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -4273,43 +3992,37 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s0
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX1032-DPP-NEXT: v_readlane_b32 s2, v3, 31
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB6_3
; GFX1032-DPP-NEXT: ; %bb.1:
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-DPP-NEXT: .LBB6_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB6_2
; GFX1032-DPP-NEXT: .LBB6_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -4335,46 +4048,44 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s[0:1]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1164-DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v2, v1
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b32 s0, s2
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB6_2
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_atomic_add_f32 v4, v0, s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_add_f32 v4, v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB6_2:
; GFX1164-DPP-NEXT: s_endpgm
;
@@ -4400,39 +4111,30 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v1, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s0, s1
+; GFX1132-DPP-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB6_2
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_atomic_add_f32 v4, v0, s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_add_f32 v3, v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB6_2:
; GFX1132-DPP-NEXT: s_endpgm
%divValue = call float @div.float.value() strictfp
@@ -4443,45 +4145,36 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_default_scope_strictfp(ptr addrspace(1) %ptr) #2 {
; GFX7LESS-LABEL: global_atomic_fadd_uni_address_uni_value_default_scope_strictfp:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT: s_mov_b32 s14, -1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0
-; GFX7LESS-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0
-; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB7_3
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dword s2, s[0:1], 0x0
-; GFX7LESS-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v1, s2
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_add_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB7_2
; GFX7LESS-NEXT: .LBB7_3:
@@ -4489,37 +4182,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_default_scop
;
; GFX9-LABEL: global_atomic_fadd_uni_address_uni_value_default_scope_strictfp:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s14, -1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-NEXT: s_add_u32 s12, s12, s11
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s6, v0
+; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB7_3
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX9-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -4532,34 +4216,27 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_default_scop
; GFX1064-LABEL: global_atomic_fadd_uni_address_uni_value_default_scope_strictfp:
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT: s_mov_b32 s14, -1
-; GFX1064-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB7_3
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1064-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -4572,75 +4249,63 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_default_scop
; GFX1032-LABEL: global_atomic_fadd_uni_address_uni_value_default_scope_strictfp:
; GFX1032: ; %bb.0:
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT: s_mov_b32 s14, -1
-; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s3, 0
+; GFX1032-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB7_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mov_b32_e32 v1, s3
-; GFX1032-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB7_2
; GFX1032-NEXT: .LBB7_3:
; GFX1032-NEXT: s_endpgm
;
; GFX1164-LABEL: global_atomic_fadd_uni_address_uni_value_default_scope_strictfp:
; GFX1164: ; %bb.0:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_clause 0x1
-; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB7_3
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v3, 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_load_b32 s2, s[0:1], 0x0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX1164-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: v_mov_b32_e32 v1, s2
; GFX1164-NEXT: s_mov_b64 s[2:3], 0
-; GFX1164-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1164-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-NEXT: v_mov_b32_e32 v1, v0
@@ -4653,85 +4318,71 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_default_scop
;
; GFX1132-LABEL: global_atomic_fadd_uni_address_uni_value_default_scope_strictfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_clause 0x1
-; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-NEXT: s_mov_b32 s3, 0
+; GFX1132-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB7_3
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v3, 0
+; GFX1132-NEXT: v_mov_b32_e32 v2, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b32 s3, s[0:1], 0x0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1132-NEXT: s_load_b32 s4, s[0:1], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mul_f32 v2, 4.0, v0
+; GFX1132-NEXT: v_mov_b32_e32 v1, s4
; GFX1132-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-NEXT: s_cbranch_execnz .LBB7_2
; GFX1132-NEXT: .LBB7_3:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_default_scope_strictfp:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT: s_mov_b32 s14, -1
; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-DPP-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0
-; GFX7LESS-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0
-; GFX7LESS-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, s2
; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
; GFX7LESS-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-DPP-NEXT: v_add_f32_e32 v0, s6, v1
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB7_2
; GFX7LESS-DPP-NEXT: .LBB7_3:
@@ -4739,37 +4390,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_default_scop
;
; GFX9-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_default_scope_strictfp:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT: s_mov_b32 s14, -1
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-DPP-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX9-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -4782,34 +4424,27 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_default_scop
; GFX1064-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_default_scope_strictfp:
; GFX1064-DPP: ; %bb.0:
; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1064-DPP-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1064-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -4822,75 +4457,63 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_default_scop
; GFX1032-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_default_scope_strictfp:
; GFX1032-DPP: ; %bb.0:
; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
+; GFX1032-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s3
-; GFX1032-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB7_2
; GFX1032-DPP-NEXT: .LBB7_3:
; GFX1032-DPP-NEXT: s_endpgm
;
; GFX1164-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_default_scope_strictfp:
; GFX1164-DPP: ; %bb.0:
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_clause 0x1
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_load_b32 s2, s[0:1], 0x0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s2
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX1164-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1164-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -4903,40 +4526,35 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_default_scop
;
; GFX1132-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_default_scope_strictfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_clause 0x1
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s3, 0
+; GFX1132-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_load_b32 s3, s[0:1], 0x0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1132-DPP-NEXT: s_load_b32 s4, s[0:1], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mul_f32 v2, 4.0, v0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1132-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB7_2
; GFX1132-DPP-NEXT: .LBB7_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -4976,24 +4594,22 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX7LESS-NEXT: .LBB8_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_brev_b32 s6, 1
+; GFX7LESS-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s2
-; GFX7LESS-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f32_e32 v2, s4, v2
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB8_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: v_readlane_b32 s3, v0, s2
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s3
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX7LESS-NEXT: v_add_f32_e32 v2, s6, v2
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v2
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB8_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB8_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
@@ -5005,15 +4621,15 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX7LESS-NEXT: .LBB8_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_add_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB8_4
; GFX7LESS-NEXT: .LBB8_5:
@@ -5049,34 +4665,34 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX9-NEXT: .LBB8_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_brev_b32 s4, 1
+; GFX9-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s4, v0, s2
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_readlane_b32 s3, v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: s_bitset0_b64 s[0:1], s2
; GFX9-NEXT: v_add_f32_e32 v2, s4, v2
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: s_cbranch_scc1 .LBB8_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9-NEXT: s_cbranch_execz .LBB8_5
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX9-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-NEXT: .LBB8_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_add_f32_e32 v0, s4, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -5116,34 +4732,33 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB8_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_brev_b32 s4, 1
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1064-NEXT: v_add_f32_e32 v2, s3, v2
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX1064-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1064-NEXT: v_add_f32_e64 v2, s4, s3
+; GFX1064-NEXT: v_readfirstlane_b32 s4, v2
; GFX1064-NEXT: s_cbranch_scc1 .LBB8_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064-NEXT: s_cbranch_execz .LBB8_5
; GFX1064-NEXT: ; %bb.3:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1064-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-NEXT: .LBB8_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_add_f32_e32 v0, s4, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -5183,38 +4798,37 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB8_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_brev_b32 s2, 1
+; GFX1032-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f32_e32 v2, s2, v2
+; GFX1032-NEXT: v_readlane_b32 s3, v0, s1
+; GFX1032-NEXT: s_bitset0_b32 s0, s1
+; GFX1032-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1032-NEXT: v_add_f32_e64 v2, s2, s3
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v2
; GFX1032-NEXT: s_cbranch_scc1 .LBB8_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1032-NEXT: s_cbranch_execz .LBB8_5
; GFX1032-NEXT: ; %bb.3:
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
+; GFX1032-NEXT: s_mov_b32 s3, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1032-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-NEXT: .LBB8_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB8_4
; GFX1032-NEXT: .LBB8_5:
; GFX1032-NEXT: s_endpgm
@@ -5239,37 +4853,36 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB8_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_brev_b32 s4, 1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f32_e32 v2, s3, v2
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: v_add_f32_e64 v2, s4, s3
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_readfirstlane_b32 s4, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB8_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: ; %bb.2:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB8_5
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v3, 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b32 v1, v3, s[0:1]
+; GFX1164-NEXT: global_load_b32 v1, v2, s[0:1]
; GFX1164-NEXT: .LBB8_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-NEXT: v_add_f32_e32 v0, s4, v1
+; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-NEXT: v_mov_b32_e32 v1, v0
@@ -5301,42 +4914,40 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB8_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: s_brev_b32 s2, 1
+; GFX1132-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f32_e32 v2, s2, v2
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: v_readlane_b32 s3, v0, s1
+; GFX1132-NEXT: s_bitset0_b32 s0, s1
+; GFX1132-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1132-NEXT: v_add_f32_e64 v2, s2, s3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: v_readfirstlane_b32 s2, v2
; GFX1132-NEXT: s_cbranch_scc1 .LBB8_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
+; GFX1132-NEXT: ; %bb.2:
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB8_5
; GFX1132-NEXT: ; %bb.3:
; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v3, 0
+; GFX1132-NEXT: v_mov_b32_e32 v2, 0
+; GFX1132-NEXT: s_mov_b32 s3, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b32 v1, v3, s[0:1]
+; GFX1132-NEXT: global_load_b32 v1, v2, s[0:1]
; GFX1132-NEXT: .LBB8_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-NEXT: s_cbranch_execnz .LBB8_4
; GFX1132-NEXT: .LBB8_5:
; GFX1132-NEXT: s_endpgm
@@ -5427,30 +5038,19 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[0:1]
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v4 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v3, v4, v3
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
@@ -5507,44 +5107,42 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1064-DPP-NEXT: ds_permute_b32 v4, v5, v3
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v3, 32
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_add_f32_e64 v3, s2, s3
+; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1064-DPP-NEXT: ; %bb.1:
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-DPP-NEXT: .LBB8_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f32_e32 v0, s4, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -5585,43 +5183,37 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s0
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX1032-DPP-NEXT: v_readlane_b32 s2, v3, 31
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1032-DPP-NEXT: ; %bb.1:
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-DPP-NEXT: .LBB8_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB8_2
; GFX1032-DPP-NEXT: .LBB8_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -5647,52 +5239,48 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s[0:1]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1164-DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v2, v1
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: v_readlane_b32 s4, v1, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b32 v5, v6, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b32 v5, v0, s[0:1]
; GFX1164-DPP-NEXT: .LBB8_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f32_e32 v4, v5, v0
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v4, v6, v[4:5], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f32_e32 v4, s4, v5
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v4, v0, v[4:5], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v4
@@ -5725,51 +5313,41 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s3, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_load_b32 v5, v6, s[0:1]
+; GFX1132-DPP-NEXT: global_load_b32 v4, v0, s[0:1]
; GFX1132-DPP-NEXT: .LBB8_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f32_e32 v4, v5, v0
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v4, v6, v[4:5], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f32_e32 v3, s2, v4
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v3, v0, v[3:4], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v5
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v5, v4
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, v3
+; GFX1132-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB8_2
; GFX1132-DPP-NEXT: .LBB8_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -5781,39 +5359,41 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 {
; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s7, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB9_3
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-NEXT: v_cvt_f64_u32_e32 v[0:1], s2
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
-; GFX7LESS-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, s6
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, s7
+; GFX7LESS-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s8
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s9
; GFX7LESS-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], s[6:7]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB9_2
; GFX7LESS-NEXT: .LBB9_3:
@@ -5821,101 +5401,107 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
;
; GFX9-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s1, v1
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB9_3
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX9-NEXT: v_cvt_f64_u32_e32 v[0:1], s2
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NEXT: v_mov_b32_e32 v3, s5
+; GFX9-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
; GFX9-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB9_2
; GFX9-NEXT: .LBB9_3:
; GFX9-NEXT: s_endpgm
;
; GFX1064-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1064: ; %bb.0:
-; GFX1064-NEXT: s_mov_b64 s[2:3], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB9_3
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: v_cvt_f64_u32_e32 v[0:1], s2
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX1064-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v2, s2
-; GFX1064-NEXT: v_mov_b32_e32 v3, s3
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, s4
+; GFX1064-NEXT: v_mov_b32_e32 v3, s5
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB9_2
; GFX1064-NEXT: .LBB9_3:
; GFX1064-NEXT: s_endpgm
;
; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: s_mov_b32 s3, exec_lo
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s6, 0
+; GFX1032-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB9_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_cvt_f64_u32_e32 v[0:1], s3
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX1032-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v2, s4
; GFX1032-NEXT: v_mov_b32_e32 v3, s5
; GFX1032-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX1032-NEXT: s_cbranch_execnz .LBB9_2
; GFX1032-NEXT: .LBB9_3:
; GFX1032-NEXT: s_endpgm
@@ -5923,37 +5509,40 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
; GFX1164-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1164: ; %bb.0:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
; GFX1164-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1164-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB9_3
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX1164-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164-NEXT: v_mov_b32_e32 v3, s3
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, s4
+; GFX1164-NEXT: v_mov_b32_e32 v3, s5
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB9_2
; GFX1164-NEXT: .LBB9_3:
; GFX1164-NEXT: s_endpgm
@@ -5961,73 +5550,77 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
; GFX1132-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1132: ; %bb.0:
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1132-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-NEXT: s_mov_b32 s6, 0
+; GFX1132-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB9_3
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
-; GFX1132-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX1132-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX1132-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s6, vcc_lo, s6
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
; GFX1132-NEXT: s_cbranch_execnz .LBB9_2
; GFX1132-NEXT: .LBB9_3:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s7, v1
; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s2
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, s6
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, s7
+; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, s8
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, s9
; GFX7LESS-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[6:7]
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v6
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v4
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v7
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v5
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB9_2
; GFX7LESS-DPP-NEXT: .LBB9_3:
@@ -6035,101 +5628,107 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
;
; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX9-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s2
-; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s7
; GFX9-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB9_2
; GFX9-DPP-NEXT: .LBB9_3:
; GFX9-DPP-NEXT: s_endpgm
;
; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1064-DPP: ; %bb.0:
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s2
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX1064-DPP-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s2
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s3
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s4
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB9_2
; GFX1064-DPP-NEXT: .LBB9_3:
; GFX1064-DPP-NEXT: s_endpgm
;
; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: s_mov_b32 s3, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s6, 0
+; GFX1032-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s4
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s5
; GFX1032-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB9_2
; GFX1032-DPP-NEXT: .LBB9_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -6137,37 +5736,40 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1164-DPP: ; %bb.0:
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1164-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX1164-DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s3
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s4
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB9_2
; GFX1164-DPP-NEXT: .LBB9_3:
; GFX1164-DPP-NEXT: s_endpgm
@@ -6175,34 +5777,36 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1132-DPP: ; %bb.0:
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s6, 0
+; GFX1132-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
-; GFX1132-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX1132-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s6, vcc_lo, s6
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB9_2
; GFX1132-DPP-NEXT: .LBB9_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -6242,50 +5846,50 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: s_mov_b32 s4, 0
+; GFX7LESS-NEXT: s_brev_b32 s5, 1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX7LESS-NEXT: .LBB10_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s4
-; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s4
-; GFX7LESS-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[4:5]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB10_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s6, s[0:1]
+; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s6
+; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s4
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, s5
+; GFX7LESS-NEXT: v_add_f64 v[3:4], s[2:3], v[3:4]
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s6
+; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v3
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s5, v4
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB10_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB10_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
-; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT: s_mov_b64 s[6:7], 0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[0:3], 0
; GFX7LESS-NEXT: .LBB10_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], s[4:5]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
-; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
+; GFX7LESS-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB10_4
; GFX7LESS-NEXT: .LBB10_5:
; GFX7LESS-NEXT: s_endpgm
@@ -6320,42 +5924,44 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX9-NEXT: .LBB10_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s3, v1, s4
-; GFX9-NEXT: v_readlane_b32 s2, v0, s4
-; GFX9-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: s_mov_b32 s0, 0
+; GFX9-NEXT: s_brev_b32 s1, 1
+; GFX9-NEXT: s_mov_b64 s[2:3], exec
+; GFX9-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9-NEXT: v_readlane_b32 s4, v0, s6
+; GFX9-NEXT: v_readlane_b32 s5, v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_add_f64 v[3:4], s[4:5], v[3:4]
+; GFX9-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v3
+; GFX9-NEXT: v_readfirstlane_b32 s1, v4
; GFX9-NEXT: s_cbranch_scc1 .LBB10_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB10_5
; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX9-NEXT: .LBB10_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB10_4
; GFX9-NEXT: .LBB10_5:
; GFX9-NEXT: s_endpgm
@@ -6390,42 +5996,42 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v4, 0
-; GFX1064-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB10_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX1064-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1064-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_mov_b32 s0, 0
+; GFX1064-NEXT: s_brev_b32 s1, 1
+; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1064-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1064-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1064-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v4
; GFX1064-NEXT: s_cbranch_scc1 .LBB10_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB10_5
; GFX1064-NEXT: ; %bb.3:
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1064-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1064-NEXT: .LBB10_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB10_4
; GFX1064-NEXT: .LBB10_5:
; GFX1064-NEXT: s_endpgm
@@ -6460,41 +6066,41 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v4, 0
-; GFX1032-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB10_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, 0
+; GFX1032-NEXT: s_brev_b32 s1, 1
+; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1032-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1032-NEXT: s_bitset0_b32 s2, s3
+; GFX1032-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v4
; GFX1032-NEXT: s_cbranch_scc1 .LBB10_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB10_5
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
+; GFX1032-NEXT: s_mov_b32 s4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1032-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1032-NEXT: .LBB10_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-NEXT: s_cbranch_execnz .LBB10_4
; GFX1032-NEXT: .LBB10_5:
; GFX1032-NEXT: s_endpgm
@@ -6519,46 +6125,46 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB10_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1164-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_mov_b32 s0, 0
+; GFX1164-NEXT: s_brev_b32 s1, 1
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s6, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1164-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1164-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v4
; GFX1164-NEXT: s_cbranch_scc1 .LBB10_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: ; %bb.2:
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB10_5
; GFX1164-NEXT: ; %bb.3:
-; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1164-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1164-NEXT: .LBB10_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB10_4
; GFX1164-NEXT: .LBB10_5:
; GFX1164-NEXT: s_endpgm
@@ -6584,44 +6190,44 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB10_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s0, 0
+; GFX1132-NEXT: s_brev_b32 s1, 1
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1132-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1132-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1132-NEXT: s_bitset0_b32 s2, s3
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v4
; GFX1132-NEXT: s_cbranch_scc1 .LBB10_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB10_5
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-NEXT: s_mov_b32 s4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1132-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1132-NEXT: .LBB10_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-NEXT: s_cbranch_execnz .LBB10_4
; GFX1132-NEXT: .LBB10_5:
; GFX1132-NEXT: s_endpgm
@@ -6712,70 +6318,72 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1]
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, v3, v1, s[0:1]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v7 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[4:5], v[6:7]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3]
+; GFX9-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX9-DPP-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_add_f64 v[9:10], v[11:12], s[0:1]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[10:11], s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v2
; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT: v_mov_b32_e32 v10, v1
; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB10_2
; GFX9-DPP-NEXT: .LBB10_3:
@@ -6812,61 +6420,69 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s[0:1]
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s5, v4, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 32
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], s[2:3], s[4:5]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v7
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: ds_permute_b32 v6, v5, v3
+; GFX1064-DPP-NEXT: ds_permute_b32 v7, v5, v4
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[6:7]
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 63
+; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX1064-DPP-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_add_f64 v[7:8], v[9:10], v[0:1]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[10:11], s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, v2
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v1
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB10_2
; GFX1064-DPP-NEXT: .LBB10_3:
; GFX1064-DPP-NEXT: s_endpgm
@@ -6901,56 +6517,58 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s2
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT: v_readlane_b32 s0, v3, 31
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v4, 31
+; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v0, s[2:3]
; GFX1032-DPP-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], v[0:1]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[7:10], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[9:10]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[1:2], v[9:10]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v1
+; GFX1032-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB10_2
; GFX1032-DPP-NEXT: .LBB10_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -6976,72 +6594,86 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s[0:1]
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_permlane64_b32 v5, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, -1, v4
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v4, 32, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v4, 4, v4
+; GFX1164-DPP-NEXT: ds_permute_b32 v5, v4, v2
+; GFX1164-DPP-NEXT: ds_permute_b32 v6, v4, v3
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[5:6]
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT: v_readlane_b32 s3, v3, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b64 v[9:10], v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[6:7], v[8:9], v[0:1]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[7:8], v[9:10], s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[7:8], v0, v[7:10], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8
; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB10_2
; GFX1164-DPP-NEXT: .LBB10_3:
; GFX1164-DPP-NEXT: s_endpgm
@@ -7067,64 +6699,64 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s2
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s2
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_readlane_b32 s0, v2, 31
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v3, 31
+; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], v[0:1]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v0, v[6:9], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB10_2
; GFX1132-DPP-NEXT: .LBB10_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -7136,48 +6768,41 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp(ptr addrspace(1) %ptr) #1 {
; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT: s_mov_b32 s14, -1
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s7, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB11_3
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
-; GFX7LESS-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
-; GFX7LESS-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, s6
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, s7
; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s8
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s9
; GFX7LESS-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], s[6:7]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB11_2
; GFX7LESS-NEXT: .LBB11_3:
@@ -7185,43 +6810,36 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_one_a
;
; GFX9-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s14, -1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-NEXT: s_add_u32 s12, s12, s11
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s1, v1
; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB11_3
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX9-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NEXT: v_mov_b32_e32 v3, s5
+; GFX9-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
; GFX9-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB11_2
; GFX9-NEXT: .LBB11_3:
; GFX9-NEXT: s_endpgm
@@ -7229,40 +6847,35 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_one_a
; GFX1064-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT: s_mov_b32 s14, -1
-; GFX1064-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB11_3
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
-; GFX1064-NEXT: s_mov_b32 s3, 0x43300000
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3]
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX1064-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v2, s2
-; GFX1064-NEXT: v_mov_b32_e32 v3, s3
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, s4
+; GFX1064-NEXT: v_mov_b32_e32 v3, s5
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB11_2
; GFX1064-NEXT: .LBB11_3:
; GFX1064-NEXT: s_endpgm
@@ -7270,171 +6883,153 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_one_a
; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1032: ; %bb.0:
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT: s_mov_b32 s14, -1
-; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s6, 0
+; GFX1032-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB11_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-NEXT: s_mov_b32 s7, 0x43300000
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v2, s4
; GFX1032-NEXT: v_mov_b32_e32 v3, s5
; GFX1032-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX1032-NEXT: s_cbranch_execnz .LBB11_2
; GFX1032-NEXT: .LBB11_3:
; GFX1032-NEXT: s_endpgm
;
; GFX1164-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1164: ; %bb.0:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_clause 0x1
-; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB11_3
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1164-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1164-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164-NEXT: v_mov_b32_e32 v3, s3
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, s4
+; GFX1164-NEXT: v_mov_b32_e32 v3, s5
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB11_2
; GFX1164-NEXT: .LBB11_3:
; GFX1164-NEXT: s_endpgm
;
; GFX1132-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_clause 0x1
-; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-NEXT: s_mov_b32 s6, 0
+; GFX1132-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v1
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB11_3
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1132-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX1132-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s6, vcc_lo, s6
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
; GFX1132-NEXT: s_cbranch_execnz .LBB11_2
; GFX1132-NEXT: .LBB11_3:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT: s_mov_b32 s14, -1
-; GFX7LESS-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-DPP-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s7, v1
; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
-; GFX7LESS-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, s6
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, s7
; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, s8
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, s9
; GFX7LESS-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[6:7]
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v6
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v4
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v7
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v5
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB11_2
; GFX7LESS-DPP-NEXT: .LBB11_3:
@@ -7442,43 +7037,36 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_one_a
;
; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT: s_mov_b32 s14, -1
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-DPP-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
-; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX9-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s7
; GFX9-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB11_2
; GFX9-DPP-NEXT: .LBB11_3:
; GFX9-DPP-NEXT: s_endpgm
@@ -7486,40 +7074,35 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_one_a
; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1064-DPP: ; %bb.0:
; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1064-DPP-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
-; GFX1064-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX1064-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s2
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s3
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s4
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB11_2
; GFX1064-DPP-NEXT: .LBB11_3:
; GFX1064-DPP-NEXT: s_endpgm
@@ -7527,123 +7110,112 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_one_a
; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1032-DPP: ; %bb.0:
; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s6, 0
+; GFX1032-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s7, 0x43300000
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s4
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s5
; GFX1032-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB11_2
; GFX1032-DPP-NEXT: .LBB11_3:
; GFX1032-DPP-NEXT: s_endpgm
;
; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1164-DPP: ; %bb.0:
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_clause 0x1
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1164-DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s3
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s4
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB11_2
; GFX1164-DPP-NEXT: .LBB11_3:
; GFX1164-DPP-NEXT: s_endpgm
;
; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_clause 0x1
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s6, 0
+; GFX1132-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX1132-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s6, vcc_lo, s6
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB11_2
; GFX1132-DPP-NEXT: .LBB11_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -7683,50 +7255,50 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: s_mov_b32 s4, 0
+; GFX7LESS-NEXT: s_brev_b32 s5, 1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX7LESS-NEXT: .LBB12_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s4
-; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s4
-; GFX7LESS-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[4:5]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB12_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s6, s[0:1]
+; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s6
+; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s4
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, s5
+; GFX7LESS-NEXT: v_add_f64 v[3:4], s[2:3], v[3:4]
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s6
+; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v3
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s5, v4
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB12_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB12_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
-; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT: s_mov_b64 s[6:7], 0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[0:3], 0
; GFX7LESS-NEXT: .LBB12_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], s[4:5]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
-; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
+; GFX7LESS-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB12_4
; GFX7LESS-NEXT: .LBB12_5:
; GFX7LESS-NEXT: s_endpgm
@@ -7761,42 +7333,44 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX9-NEXT: .LBB12_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s3, v1, s4
-; GFX9-NEXT: v_readlane_b32 s2, v0, s4
-; GFX9-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: s_mov_b32 s0, 0
+; GFX9-NEXT: s_brev_b32 s1, 1
+; GFX9-NEXT: s_mov_b64 s[2:3], exec
+; GFX9-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9-NEXT: v_readlane_b32 s4, v0, s6
+; GFX9-NEXT: v_readlane_b32 s5, v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_add_f64 v[3:4], s[4:5], v[3:4]
+; GFX9-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v3
+; GFX9-NEXT: v_readfirstlane_b32 s1, v4
; GFX9-NEXT: s_cbranch_scc1 .LBB12_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB12_5
; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX9-NEXT: .LBB12_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB12_4
; GFX9-NEXT: .LBB12_5:
; GFX9-NEXT: s_endpgm
@@ -7831,42 +7405,42 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v4, 0
-; GFX1064-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB12_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX1064-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1064-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_mov_b32 s0, 0
+; GFX1064-NEXT: s_brev_b32 s1, 1
+; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1064-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1064-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1064-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v4
; GFX1064-NEXT: s_cbranch_scc1 .LBB12_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB12_5
; GFX1064-NEXT: ; %bb.3:
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1064-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1064-NEXT: .LBB12_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB12_4
; GFX1064-NEXT: .LBB12_5:
; GFX1064-NEXT: s_endpgm
@@ -7901,41 +7475,41 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v4, 0
-; GFX1032-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB12_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, 0
+; GFX1032-NEXT: s_brev_b32 s1, 1
+; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1032-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1032-NEXT: s_bitset0_b32 s2, s3
+; GFX1032-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v4
; GFX1032-NEXT: s_cbranch_scc1 .LBB12_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB12_5
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
+; GFX1032-NEXT: s_mov_b32 s4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1032-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1032-NEXT: .LBB12_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-NEXT: s_cbranch_execnz .LBB12_4
; GFX1032-NEXT: .LBB12_5:
; GFX1032-NEXT: s_endpgm
@@ -7960,46 +7534,46 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB12_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1164-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_mov_b32 s0, 0
+; GFX1164-NEXT: s_brev_b32 s1, 1
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s6, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1164-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1164-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v4
; GFX1164-NEXT: s_cbranch_scc1 .LBB12_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: ; %bb.2:
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB12_5
; GFX1164-NEXT: ; %bb.3:
-; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1164-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1164-NEXT: .LBB12_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB12_4
; GFX1164-NEXT: .LBB12_5:
; GFX1164-NEXT: s_endpgm
@@ -8025,44 +7599,44 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB12_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s0, 0
+; GFX1132-NEXT: s_brev_b32 s1, 1
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1132-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1132-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1132-NEXT: s_bitset0_b32 s2, s3
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v4
; GFX1132-NEXT: s_cbranch_scc1 .LBB12_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB12_5
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-NEXT: s_mov_b32 s4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1132-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1132-NEXT: .LBB12_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-NEXT: s_cbranch_execnz .LBB12_4
; GFX1132-NEXT: .LBB12_5:
; GFX1132-NEXT: s_endpgm
@@ -8153,70 +7727,72 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1]
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, v3, v1, s[0:1]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v7 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[4:5], v[6:7]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB12_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3]
+; GFX9-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX9-DPP-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_add_f64 v[9:10], v[11:12], s[0:1]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[10:11], s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v2
; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT: v_mov_b32_e32 v10, v1
; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB12_2
; GFX9-DPP-NEXT: .LBB12_3:
@@ -8253,61 +7829,69 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s[0:1]
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s5, v4, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 32
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], s[2:3], s[4:5]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v7
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: ds_permute_b32 v6, v5, v3
+; GFX1064-DPP-NEXT: ds_permute_b32 v7, v5, v4
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[6:7]
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 63
+; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB12_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX1064-DPP-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_add_f64 v[7:8], v[9:10], v[0:1]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[10:11], s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, v2
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v1
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB12_2
; GFX1064-DPP-NEXT: .LBB12_3:
; GFX1064-DPP-NEXT: s_endpgm
@@ -8342,56 +7926,58 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s2
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT: v_readlane_b32 s0, v3, 31
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v4, 31
+; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB12_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v0, s[2:3]
; GFX1032-DPP-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], v[0:1]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[7:10], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[9:10]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[1:2], v[9:10]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v1
+; GFX1032-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB12_2
; GFX1032-DPP-NEXT: .LBB12_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -8417,72 +8003,86 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s[0:1]
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_permlane64_b32 v5, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, -1, v4
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v4, 32, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v4, 4, v4
+; GFX1164-DPP-NEXT: ds_permute_b32 v5, v4, v2
+; GFX1164-DPP-NEXT: ds_permute_b32 v6, v4, v3
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[5:6]
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT: v_readlane_b32 s3, v3, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB12_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b64 v[9:10], v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[6:7], v[8:9], v[0:1]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[7:8], v[9:10], s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[7:8], v0, v[7:10], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8
; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB12_2
; GFX1164-DPP-NEXT: .LBB12_3:
; GFX1164-DPP-NEXT: s_endpgm
@@ -8508,64 +8108,64 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s2
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s2
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_readlane_b32 s0, v2, 31
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v3, 31
+; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB12_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], v[0:1]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v0, v[6:9], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB12_2
; GFX1132-DPP-NEXT: .LBB12_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -8577,48 +8177,41 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp(ptr addrspace(1) %ptr) #2{
; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT: s_mov_b32 s14, -1
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s7, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB13_3
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
-; GFX7LESS-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
-; GFX7LESS-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, s6
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, s7
; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s8
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s9
; GFX7LESS-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], s[6:7]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB13_2
; GFX7LESS-NEXT: .LBB13_3:
@@ -8626,43 +8219,36 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
;
; GFX9-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s14, -1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-NEXT: s_add_u32 s12, s12, s11
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s1, v1
; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB13_3
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX9-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NEXT: v_mov_b32_e32 v3, s5
+; GFX9-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
; GFX9-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB13_2
; GFX9-NEXT: .LBB13_3:
; GFX9-NEXT: s_endpgm
@@ -8670,40 +8256,35 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
; GFX1064-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT: s_mov_b32 s14, -1
-; GFX1064-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB13_3
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
-; GFX1064-NEXT: s_mov_b32 s3, 0x43300000
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3]
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX1064-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v2, s2
-; GFX1064-NEXT: v_mov_b32_e32 v3, s3
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, s4
+; GFX1064-NEXT: v_mov_b32_e32 v3, s5
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB13_2
; GFX1064-NEXT: .LBB13_3:
; GFX1064-NEXT: s_endpgm
@@ -8711,171 +8292,153 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1032: ; %bb.0:
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT: s_mov_b32 s14, -1
-; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s6, 0
+; GFX1032-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB13_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-NEXT: s_mov_b32 s7, 0x43300000
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v2, s4
; GFX1032-NEXT: v_mov_b32_e32 v3, s5
; GFX1032-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX1032-NEXT: s_cbranch_execnz .LBB13_2
; GFX1032-NEXT: .LBB13_3:
; GFX1032-NEXT: s_endpgm
;
; GFX1164-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1164: ; %bb.0:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_clause 0x1
-; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB13_3
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1164-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1164-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164-NEXT: v_mov_b32_e32 v3, s3
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, s4
+; GFX1164-NEXT: v_mov_b32_e32 v3, s5
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB13_2
; GFX1164-NEXT: .LBB13_3:
; GFX1164-NEXT: s_endpgm
;
; GFX1132-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_clause 0x1
-; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-NEXT: s_mov_b32 s6, 0
+; GFX1132-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v1
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB13_3
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1132-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX1132-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s6, vcc_lo, s6
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
; GFX1132-NEXT: s_cbranch_execnz .LBB13_2
; GFX1132-NEXT: .LBB13_3:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT: s_mov_b32 s14, -1
-; GFX7LESS-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-DPP-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s7, v1
; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB13_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
-; GFX7LESS-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, s6
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, s7
; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, s8
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, s9
; GFX7LESS-DPP-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[6:7]
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v6
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v4
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v7
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v5
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB13_2
; GFX7LESS-DPP-NEXT: .LBB13_3:
@@ -8883,43 +8446,36 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
;
; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT: s_mov_b32 s14, -1
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-DPP-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB13_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
-; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX9-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s7
; GFX9-DPP-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB13_2
; GFX9-DPP-NEXT: .LBB13_3:
; GFX9-DPP-NEXT: s_endpgm
@@ -8927,40 +8483,35 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1064-DPP: ; %bb.0:
; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1064-DPP-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB13_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
-; GFX1064-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX1064-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s2
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s3
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s4
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB13_2
; GFX1064-DPP-NEXT: .LBB13_3:
; GFX1064-DPP-NEXT: s_endpgm
@@ -8968,123 +8519,112 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1032-DPP: ; %bb.0:
; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s6, 0
+; GFX1032-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB13_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s7, 0x43300000
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s4
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s5
; GFX1032-DPP-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB13_2
; GFX1032-DPP-NEXT: .LBB13_3:
; GFX1032-DPP-NEXT: s_endpgm
;
; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1164-DPP: ; %bb.0:
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_clause 0x1
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB13_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1164-DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s3
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s4
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB13_2
; GFX1164-DPP-NEXT: .LBB13_3:
; GFX1164-DPP-NEXT: s_endpgm
;
; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_clause 0x1
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s6, 0
+; GFX1132-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB13_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX1132-DPP-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s6, vcc_lo, s6
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB13_2
; GFX1132-DPP-NEXT: .LBB13_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -9124,50 +8664,50 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: s_mov_b32 s4, 0
+; GFX7LESS-NEXT: s_brev_b32 s5, 1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX7LESS-NEXT: .LBB14_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s4
-; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s4
-; GFX7LESS-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[4:5]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB14_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s6, s[0:1]
+; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s6
+; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s4
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, s5
+; GFX7LESS-NEXT: v_add_f64 v[3:4], s[2:3], v[3:4]
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s6
+; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v3
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s5, v4
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB14_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB14_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
-; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT: s_mov_b64 s[6:7], 0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[0:3], 0
; GFX7LESS-NEXT: .LBB14_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], s[4:5]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
-; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
+; GFX7LESS-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB14_4
; GFX7LESS-NEXT: .LBB14_5:
; GFX7LESS-NEXT: s_endpgm
@@ -9202,42 +8742,44 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX9-NEXT: .LBB14_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s3, v1, s4
-; GFX9-NEXT: v_readlane_b32 s2, v0, s4
-; GFX9-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: s_mov_b32 s0, 0
+; GFX9-NEXT: s_brev_b32 s1, 1
+; GFX9-NEXT: s_mov_b64 s[2:3], exec
+; GFX9-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9-NEXT: v_readlane_b32 s4, v0, s6
+; GFX9-NEXT: v_readlane_b32 s5, v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_add_f64 v[3:4], s[4:5], v[3:4]
+; GFX9-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v3
+; GFX9-NEXT: v_readfirstlane_b32 s1, v4
; GFX9-NEXT: s_cbranch_scc1 .LBB14_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB14_5
; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX9-NEXT: .LBB14_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB14_4
; GFX9-NEXT: .LBB14_5:
; GFX9-NEXT: s_endpgm
@@ -9272,42 +8814,42 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v4, 0
-; GFX1064-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB14_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX1064-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1064-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_mov_b32 s0, 0
+; GFX1064-NEXT: s_brev_b32 s1, 1
+; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1064-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1064-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1064-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v4
; GFX1064-NEXT: s_cbranch_scc1 .LBB14_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB14_5
; GFX1064-NEXT: ; %bb.3:
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1064-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1064-NEXT: .LBB14_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB14_4
; GFX1064-NEXT: .LBB14_5:
; GFX1064-NEXT: s_endpgm
@@ -9342,41 +8884,41 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v4, 0
-; GFX1032-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB14_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, 0
+; GFX1032-NEXT: s_brev_b32 s1, 1
+; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1032-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1032-NEXT: s_bitset0_b32 s2, s3
+; GFX1032-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v4
; GFX1032-NEXT: s_cbranch_scc1 .LBB14_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB14_5
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
+; GFX1032-NEXT: s_mov_b32 s4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1032-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1032-NEXT: .LBB14_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-NEXT: s_cbranch_execnz .LBB14_4
; GFX1032-NEXT: .LBB14_5:
; GFX1032-NEXT: s_endpgm
@@ -9401,46 +8943,46 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB14_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1164-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_mov_b32 s0, 0
+; GFX1164-NEXT: s_brev_b32 s1, 1
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s6, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1164-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1164-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v4
; GFX1164-NEXT: s_cbranch_scc1 .LBB14_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: ; %bb.2:
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB14_5
; GFX1164-NEXT: ; %bb.3:
-; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1164-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1164-NEXT: .LBB14_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB14_4
; GFX1164-NEXT: .LBB14_5:
; GFX1164-NEXT: s_endpgm
@@ -9466,44 +9008,44 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB14_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s0, 0
+; GFX1132-NEXT: s_brev_b32 s1, 1
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1132-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1132-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1132-NEXT: s_bitset0_b32 s2, s3
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v4
; GFX1132-NEXT: s_cbranch_scc1 .LBB14_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB14_5
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-NEXT: s_mov_b32 s4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1132-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1132-NEXT: .LBB14_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-NEXT: s_cbranch_execnz .LBB14_4
; GFX1132-NEXT: .LBB14_5:
; GFX1132-NEXT: s_endpgm
@@ -9594,70 +9136,72 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1]
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, v3, v1, s[0:1]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v7 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[4:5], v[6:7]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB14_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3]
+; GFX9-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX9-DPP-NEXT: .LBB14_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_add_f64 v[9:10], v[11:12], s[0:1]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[10:11], s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v2
; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT: v_mov_b32_e32 v10, v1
; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB14_2
; GFX9-DPP-NEXT: .LBB14_3:
@@ -9694,61 +9238,69 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s[0:1]
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s5, v4, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 32
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], s[2:3], s[4:5]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v7
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: ds_permute_b32 v6, v5, v3
+; GFX1064-DPP-NEXT: ds_permute_b32 v7, v5, v4
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[6:7]
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 63
+; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB14_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX1064-DPP-NEXT: .LBB14_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_add_f64 v[7:8], v[9:10], v[0:1]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[10:11], s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, v2
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v1
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB14_2
; GFX1064-DPP-NEXT: .LBB14_3:
; GFX1064-DPP-NEXT: s_endpgm
@@ -9783,56 +9335,58 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s2
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT: v_readlane_b32 s0, v3, 31
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v4, 31
+; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB14_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v0, s[2:3]
; GFX1032-DPP-NEXT: .LBB14_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], v[0:1]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[7:10], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[9:10]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[1:2], v[9:10]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v1
+; GFX1032-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB14_2
; GFX1032-DPP-NEXT: .LBB14_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -9858,72 +9412,86 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s[0:1]
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_permlane64_b32 v5, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, -1, v4
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v4, 32, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v4, 4, v4
+; GFX1164-DPP-NEXT: ds_permute_b32 v5, v4, v2
+; GFX1164-DPP-NEXT: ds_permute_b32 v6, v4, v3
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[5:6]
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT: v_readlane_b32 s3, v3, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB14_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b64 v[9:10], v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB14_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[6:7], v[8:9], v[0:1]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[7:8], v[9:10], s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[7:8], v0, v[7:10], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8
; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB14_2
; GFX1164-DPP-NEXT: .LBB14_3:
; GFX1164-DPP-NEXT: s_endpgm
@@ -9949,64 +9517,64 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s2
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s2
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_readlane_b32 s0, v2, 31
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v3, 31
+; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB14_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB14_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], v[0:1]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v0, v[6:9], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB14_2
; GFX1132-DPP-NEXT: .LBB14_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -10047,50 +9615,50 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: s_mov_b32 s4, 0
+; GFX7LESS-NEXT: s_brev_b32 s5, 1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX7LESS-NEXT: .LBB15_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s4
-; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s4
-; GFX7LESS-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[4:5]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB15_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s6, s[0:1]
+; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s6
+; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s4
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, s5
+; GFX7LESS-NEXT: v_add_f64 v[3:4], s[2:3], v[3:4]
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s6
+; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v3
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s5, v4
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB15_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB15_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
-; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT: s_mov_b64 s[6:7], 0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[0:3], 0
; GFX7LESS-NEXT: .LBB15_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], s[4:5]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
-; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
+; GFX7LESS-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB15_4
; GFX7LESS-NEXT: .LBB15_5:
; GFX7LESS-NEXT: s_endpgm
@@ -10125,42 +9693,44 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX9-NEXT: .LBB15_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s3, v1, s4
-; GFX9-NEXT: v_readlane_b32 s2, v0, s4
-; GFX9-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: s_mov_b32 s0, 0
+; GFX9-NEXT: s_brev_b32 s1, 1
+; GFX9-NEXT: s_mov_b64 s[2:3], exec
+; GFX9-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9-NEXT: v_readlane_b32 s4, v0, s6
+; GFX9-NEXT: v_readlane_b32 s5, v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_add_f64 v[3:4], s[4:5], v[3:4]
+; GFX9-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v3
+; GFX9-NEXT: v_readfirstlane_b32 s1, v4
; GFX9-NEXT: s_cbranch_scc1 .LBB15_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB15_5
; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX9-NEXT: .LBB15_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB15_4
; GFX9-NEXT: .LBB15_5:
; GFX9-NEXT: s_endpgm
@@ -10195,42 +9765,42 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v4, 0
-; GFX1064-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB15_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX1064-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1064-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_mov_b32 s0, 0
+; GFX1064-NEXT: s_brev_b32 s1, 1
+; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1064-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1064-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1064-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v4
; GFX1064-NEXT: s_cbranch_scc1 .LBB15_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB15_5
; GFX1064-NEXT: ; %bb.3:
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1064-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1064-NEXT: .LBB15_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB15_4
; GFX1064-NEXT: .LBB15_5:
; GFX1064-NEXT: s_endpgm
@@ -10265,41 +9835,41 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v4, 0
-; GFX1032-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB15_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, 0
+; GFX1032-NEXT: s_brev_b32 s1, 1
+; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1032-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1032-NEXT: s_bitset0_b32 s2, s3
+; GFX1032-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v4
; GFX1032-NEXT: s_cbranch_scc1 .LBB15_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB15_5
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
+; GFX1032-NEXT: s_mov_b32 s4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1032-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1032-NEXT: .LBB15_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-NEXT: s_cbranch_execnz .LBB15_4
; GFX1032-NEXT: .LBB15_5:
; GFX1032-NEXT: s_endpgm
@@ -10324,46 +9894,46 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB15_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1164-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_mov_b32 s0, 0
+; GFX1164-NEXT: s_brev_b32 s1, 1
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s6, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1164-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1164-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v4
; GFX1164-NEXT: s_cbranch_scc1 .LBB15_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: ; %bb.2:
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB15_5
; GFX1164-NEXT: ; %bb.3:
-; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1164-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1164-NEXT: .LBB15_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB15_4
; GFX1164-NEXT: .LBB15_5:
; GFX1164-NEXT: s_endpgm
@@ -10389,44 +9959,44 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB15_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s0, 0
+; GFX1132-NEXT: s_brev_b32 s1, 1
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1132-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1132-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1132-NEXT: s_bitset0_b32 s2, s3
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v4
; GFX1132-NEXT: s_cbranch_scc1 .LBB15_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB15_5
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-NEXT: s_mov_b32 s4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1132-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1132-NEXT: .LBB15_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-NEXT: s_cbranch_execnz .LBB15_4
; GFX1132-NEXT: .LBB15_5:
; GFX1132-NEXT: s_endpgm
@@ -10517,70 +10087,72 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1]
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, v3, v1, s[0:1]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v7 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[4:5], v[6:7]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB15_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3]
+; GFX9-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX9-DPP-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_add_f64 v[9:10], v[11:12], s[0:1]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[10:11], s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v2
; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT: v_mov_b32_e32 v10, v1
; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB15_2
; GFX9-DPP-NEXT: .LBB15_3:
@@ -10617,61 +10189,69 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s[0:1]
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s5, v4, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 32
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], s[2:3], s[4:5]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v7
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: ds_permute_b32 v6, v5, v3
+; GFX1064-DPP-NEXT: ds_permute_b32 v7, v5, v4
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[6:7]
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 63
+; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB15_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX1064-DPP-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_add_f64 v[7:8], v[9:10], v[0:1]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[10:11], s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, v2
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v1
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB15_2
; GFX1064-DPP-NEXT: .LBB15_3:
; GFX1064-DPP-NEXT: s_endpgm
@@ -10706,56 +10286,58 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s2
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT: v_readlane_b32 s0, v3, 31
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v4, 31
+; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB15_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v0, s[2:3]
; GFX1032-DPP-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], v[0:1]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[7:10], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[9:10]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[1:2], v[9:10]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v1
+; GFX1032-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB15_2
; GFX1032-DPP-NEXT: .LBB15_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -10781,72 +10363,86 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s[0:1]
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_permlane64_b32 v5, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, -1, v4
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v4, 32, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v4, 4, v4
+; GFX1164-DPP-NEXT: ds_permute_b32 v5, v4, v2
+; GFX1164-DPP-NEXT: ds_permute_b32 v6, v4, v3
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[5:6]
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT: v_readlane_b32 s3, v3, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB15_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b64 v[9:10], v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[6:7], v[8:9], v[0:1]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[7:8], v[9:10], s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[7:8], v0, v[7:10], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8
; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB15_2
; GFX1164-DPP-NEXT: .LBB15_3:
; GFX1164-DPP-NEXT: s_endpgm
@@ -10872,64 +10468,64 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s2
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s2
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_readlane_b32 s0, v2, 31
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v3, 31
+; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB15_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], v[0:1]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v0, v[6:9], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB15_2
; GFX1132-DPP-NEXT: .LBB15_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -10941,48 +10537,41 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp(ptr addrspace(1) %ptr) #2 {
; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT: s_mov_b32 s14, -1
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s7, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB16_3
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
-; GFX7LESS-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
-; GFX7LESS-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, s6
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, s7
; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s8
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s9
; GFX7LESS-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], s[6:7]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB16_2
; GFX7LESS-NEXT: .LBB16_3:
@@ -10990,43 +10579,36 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
;
; GFX9-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s14, -1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-NEXT: s_add_u32 s12, s12, s11
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s1, v1
; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB16_3
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX9-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NEXT: v_mov_b32_e32 v3, s5
+; GFX9-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
; GFX9-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB16_2
; GFX9-NEXT: .LBB16_3:
; GFX9-NEXT: s_endpgm
@@ -11034,40 +10616,35 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
; GFX1064-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT: s_mov_b32 s14, -1
-; GFX1064-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB16_3
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
-; GFX1064-NEXT: s_mov_b32 s3, 0x43300000
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3]
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX1064-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v2, s2
-; GFX1064-NEXT: v_mov_b32_e32 v3, s3
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, s4
+; GFX1064-NEXT: v_mov_b32_e32 v3, s5
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB16_2
; GFX1064-NEXT: .LBB16_3:
; GFX1064-NEXT: s_endpgm
@@ -11075,171 +10652,153 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
; GFX1032: ; %bb.0:
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT: s_mov_b32 s14, -1
-; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s6, 0
+; GFX1032-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB16_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-NEXT: s_mov_b32 s7, 0x43300000
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v2, s4
; GFX1032-NEXT: v_mov_b32_e32 v3, s5
; GFX1032-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX1032-NEXT: s_cbranch_execnz .LBB16_2
; GFX1032-NEXT: .LBB16_3:
; GFX1032-NEXT: s_endpgm
;
; GFX1164-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
; GFX1164: ; %bb.0:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_clause 0x1
-; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB16_3
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1164-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1164-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164-NEXT: v_mov_b32_e32 v3, s3
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, s4
+; GFX1164-NEXT: v_mov_b32_e32 v3, s5
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB16_2
; GFX1164-NEXT: .LBB16_3:
; GFX1164-NEXT: s_endpgm
;
; GFX1132-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_clause 0x1
-; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-NEXT: s_mov_b32 s6, 0
+; GFX1132-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v1
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB16_3
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1132-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX1132-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s6, vcc_lo, s6
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
; GFX1132-NEXT: s_cbranch_execnz .LBB16_2
; GFX1132-NEXT: .LBB16_3:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT: s_mov_b32 s14, -1
-; GFX7LESS-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-DPP-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s7, v1
; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB16_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
-; GFX7LESS-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, s6
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, s7
; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, s8
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, s9
; GFX7LESS-DPP-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[6:7]
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v6
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v4
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v7
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v5
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB16_2
; GFX7LESS-DPP-NEXT: .LBB16_3:
@@ -11247,43 +10806,36 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
;
; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT: s_mov_b32 s14, -1
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-DPP-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB16_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
-; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX9-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s7
; GFX9-DPP-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB16_2
; GFX9-DPP-NEXT: .LBB16_3:
; GFX9-DPP-NEXT: s_endpgm
@@ -11291,40 +10843,35 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
; GFX1064-DPP: ; %bb.0:
; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1064-DPP-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB16_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
-; GFX1064-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX1064-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s2
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s3
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s4
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB16_2
; GFX1064-DPP-NEXT: .LBB16_3:
; GFX1064-DPP-NEXT: s_endpgm
@@ -11332,123 +10879,112 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
; GFX1032-DPP: ; %bb.0:
; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s6, 0
+; GFX1032-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB16_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s7, 0x43300000
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s4
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s5
; GFX1032-DPP-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB16_2
; GFX1032-DPP-NEXT: .LBB16_3:
; GFX1032-DPP-NEXT: s_endpgm
;
; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
; GFX1164-DPP: ; %bb.0:
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_clause 0x1
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB16_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1164-DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s3
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s4
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB16_2
; GFX1164-DPP-NEXT: .LBB16_3:
; GFX1164-DPP-NEXT: s_endpgm
;
; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_clause 0x1
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s6, 0
+; GFX1132-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB16_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX1132-DPP-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s6, vcc_lo, s6
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB16_2
; GFX1132-DPP-NEXT: .LBB16_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -11488,50 +11024,50 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: s_mov_b32 s4, 0
+; GFX7LESS-NEXT: s_brev_b32 s5, 1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX7LESS-NEXT: .LBB17_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s4
-; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s4
-; GFX7LESS-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[4:5]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB17_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s6, s[0:1]
+; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s6
+; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s4
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, s5
+; GFX7LESS-NEXT: v_add_f64 v[3:4], s[2:3], v[3:4]
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s6
+; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v3
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s5, v4
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB17_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB17_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
-; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT: s_mov_b64 s[6:7], 0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[0:3], 0
; GFX7LESS-NEXT: .LBB17_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], s[4:5]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
-; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
+; GFX7LESS-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB17_4
; GFX7LESS-NEXT: .LBB17_5:
; GFX7LESS-NEXT: s_endpgm
@@ -11566,42 +11102,44 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX9-NEXT: .LBB17_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s3, v1, s4
-; GFX9-NEXT: v_readlane_b32 s2, v0, s4
-; GFX9-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: s_mov_b32 s0, 0
+; GFX9-NEXT: s_brev_b32 s1, 1
+; GFX9-NEXT: s_mov_b64 s[2:3], exec
+; GFX9-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9-NEXT: v_readlane_b32 s4, v0, s6
+; GFX9-NEXT: v_readlane_b32 s5, v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_add_f64 v[3:4], s[4:5], v[3:4]
+; GFX9-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v3
+; GFX9-NEXT: v_readfirstlane_b32 s1, v4
; GFX9-NEXT: s_cbranch_scc1 .LBB17_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB17_5
; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX9-NEXT: .LBB17_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB17_4
; GFX9-NEXT: .LBB17_5:
; GFX9-NEXT: s_endpgm
@@ -11636,42 +11174,42 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v4, 0
-; GFX1064-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB17_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX1064-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1064-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_mov_b32 s0, 0
+; GFX1064-NEXT: s_brev_b32 s1, 1
+; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1064-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1064-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1064-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v4
; GFX1064-NEXT: s_cbranch_scc1 .LBB17_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB17_5
; GFX1064-NEXT: ; %bb.3:
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1064-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1064-NEXT: .LBB17_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB17_4
; GFX1064-NEXT: .LBB17_5:
; GFX1064-NEXT: s_endpgm
@@ -11706,41 +11244,41 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v4, 0
-; GFX1032-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB17_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, 0
+; GFX1032-NEXT: s_brev_b32 s1, 1
+; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1032-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1032-NEXT: s_bitset0_b32 s2, s3
+; GFX1032-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v4
; GFX1032-NEXT: s_cbranch_scc1 .LBB17_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB17_5
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
+; GFX1032-NEXT: s_mov_b32 s4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1032-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1032-NEXT: .LBB17_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-NEXT: s_cbranch_execnz .LBB17_4
; GFX1032-NEXT: .LBB17_5:
; GFX1032-NEXT: s_endpgm
@@ -11765,46 +11303,46 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB17_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1164-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_mov_b32 s0, 0
+; GFX1164-NEXT: s_brev_b32 s1, 1
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s6, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1164-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1164-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v4
; GFX1164-NEXT: s_cbranch_scc1 .LBB17_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: ; %bb.2:
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB17_5
; GFX1164-NEXT: ; %bb.3:
-; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1164-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1164-NEXT: .LBB17_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB17_4
; GFX1164-NEXT: .LBB17_5:
; GFX1164-NEXT: s_endpgm
@@ -11830,44 +11368,44 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB17_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s0, 0
+; GFX1132-NEXT: s_brev_b32 s1, 1
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1132-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1132-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1132-NEXT: s_bitset0_b32 s2, s3
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v4
; GFX1132-NEXT: s_cbranch_scc1 .LBB17_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB17_5
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-NEXT: s_mov_b32 s4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1132-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1132-NEXT: .LBB17_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-NEXT: s_cbranch_execnz .LBB17_4
; GFX1132-NEXT: .LBB17_5:
; GFX1132-NEXT: s_endpgm
@@ -11958,70 +11496,72 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1]
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, v3, v1, s[0:1]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v7 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[4:5], v[6:7]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB17_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3]
+; GFX9-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX9-DPP-NEXT: .LBB17_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_add_f64 v[9:10], v[11:12], s[0:1]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[10:11], s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v2
; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT: v_mov_b32_e32 v10, v1
; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB17_2
; GFX9-DPP-NEXT: .LBB17_3:
@@ -12058,61 +11598,69 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s[0:1]
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s5, v4, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 32
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], s[2:3], s[4:5]
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v7
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: ds_permute_b32 v6, v5, v3
+; GFX1064-DPP-NEXT: ds_permute_b32 v7, v5, v4
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[6:7]
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 63
+; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB17_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX1064-DPP-NEXT: .LBB17_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_add_f64 v[7:8], v[9:10], v[0:1]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[10:11], s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, v2
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v1
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB17_2
; GFX1064-DPP-NEXT: .LBB17_3:
; GFX1064-DPP-NEXT: s_endpgm
@@ -12147,56 +11695,58 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s2
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT: v_readlane_b32 s0, v3, 31
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v4, 31
+; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB17_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v0, s[2:3]
; GFX1032-DPP-NEXT: .LBB17_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], v[0:1]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[7:10], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[9:10]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[1:2], v[9:10]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v1
+; GFX1032-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB17_2
; GFX1032-DPP-NEXT: .LBB17_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -12222,72 +11772,86 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s[0:1]
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_permlane64_b32 v5, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, -1, v4
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v4, 32, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v4, 4, v4
+; GFX1164-DPP-NEXT: ds_permute_b32 v5, v4, v2
+; GFX1164-DPP-NEXT: ds_permute_b32 v6, v4, v3
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[5:6]
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT: v_readlane_b32 s3, v3, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB17_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b64 v[9:10], v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB17_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[6:7], v[8:9], v[0:1]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[7:8], v[9:10], s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[7:8], v0, v[7:10], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8
; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB17_2
; GFX1164-DPP-NEXT: .LBB17_3:
; GFX1164-DPP-NEXT: s_endpgm
@@ -12313,64 +11877,64 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s2
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s2
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_readlane_b32 s0, v2, 31
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v3, 31
+; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB17_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB17_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], v[0:1]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v0, v[6:9], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB17_2
; GFX1132-DPP-NEXT: .LBB17_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -12382,35 +11946,36 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr) {
; GFX7LESS-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB18_3
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX7LESS-NEXT: v_mul_f32_e32 v2, 4.0, v0
-; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s2
+; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: .LBB18_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_add_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB18_2
; GFX7LESS-NEXT: .LBB18_3:
@@ -12418,27 +11983,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
;
; GFX9-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s6, v0
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB18_3
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
-; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s4
; GFX9-NEXT: .LBB18_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -12450,27 +12016,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
;
; GFX1064-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1064: ; %bb.0:
-; GFX1064-NEXT: s_mov_b64 s[2:3], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-NEXT: v_readfirstlane_b32 s6, v0
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB18_3
; GFX1064-NEXT: ; %bb.1:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
-; GFX1064-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX1064-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v1, s4
+; GFX1064-NEXT: v_mov_b32_e32 v1, s2
+; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: .LBB18_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -12482,31 +12049,32 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
;
; GFX1032-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: s_mov_b32 s3, exec_lo
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s3, 0
+; GFX1032-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB18_3
; GFX1032-NEXT: ; %bb.1:
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
-; GFX1032-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-NEXT: .LBB18_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB18_2
; GFX1032-NEXT: .LBB18_3:
; GFX1032-NEXT: s_endpgm
@@ -12514,74 +12082,78 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
; GFX1164-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1164: ; %bb.0:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
; GFX1164-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1164-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB18_2
; GFX1164-NEXT: ; %bb.1:
; GFX1164-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164-NEXT: v_cvt_f32_ubyte0_e32 v0, s0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-NEXT: v_mov_b32_e32 v1, s0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_atomic_add_f32 v1, v0, s[2:3]
+; GFX1164-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GFX1164-NEXT: .LBB18_2:
; GFX1164-NEXT: s_endpgm
;
; GFX1132-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1132: ; %bb.0:
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
; GFX1132-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1132-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB18_2
; GFX1132-NEXT: ; %bb.1:
; GFX1132-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_cvt_f32_ubyte0_e32 v0, s0
-; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mul_f32 v0, 4.0, v0
+; GFX1132-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_atomic_add_f32 v1, v0, s[2:3]
+; GFX1132-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GFX1132-NEXT: .LBB18_2:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB18_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
-; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, s6
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, s2
+; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
; GFX7LESS-DPP-NEXT: .LBB18_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-DPP-NEXT: v_add_f32_e32 v0, s6, v1
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB18_2
; GFX7LESS-DPP-NEXT: .LBB18_3:
@@ -12589,27 +12161,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
;
; GFX9-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s6, v0
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB18_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
-; GFX9-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX9-DPP-NEXT: .LBB18_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -12621,27 +12194,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
;
; GFX1064-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1064-DPP: ; %bb.0:
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s6, v0
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB18_3
; GFX1064-DPP-NEXT: ; %bb.1:
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX1064-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX1064-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s4
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s2
+; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: .LBB18_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -12653,31 +12227,32 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
;
; GFX1032-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: s_mov_b32 s3, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
+; GFX1032-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s2, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB18_3
; GFX1032-DPP-NEXT: ; %bb.1:
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
-; GFX1032-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-DPP-NEXT: .LBB18_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB18_2
; GFX1032-DPP-NEXT: .LBB18_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -12685,40 +12260,43 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
; GFX1164-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1164-DPP: ; %bb.0:
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1164-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB18_2
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_atomic_add_f32 v1, v0, s[2:3]
+; GFX1164-DPP-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GFX1164-DPP-NEXT: .LBB18_2:
; GFX1164-DPP-NEXT: s_endpgm
;
; GFX1132-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1132-DPP: ; %bb.0:
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
; GFX1132-DPP-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1132-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB18_2
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s0
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mul_f32 v0, 4.0, v0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_atomic_add_f32 v1, v0, s[2:3]
+; GFX1132-DPP-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GFX1132-DPP-NEXT: .LBB18_2:
; GFX1132-DPP-NEXT: s_endpgm
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 monotonic, align 8, !amdgpu.no.fine.grained.memory !1, !amdgpu.no.remote.memory !1, !amdgpu.ignore.denormal.mode !1
@@ -12728,35 +12306,36 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr) {
; GFX7LESS-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB19_3
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX7LESS-NEXT: v_mul_f32_e32 v2, 4.0, v0
-; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s2
+; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: .LBB19_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_add_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB19_2
; GFX7LESS-NEXT: .LBB19_3:
@@ -12764,27 +12343,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
;
; GFX9-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s6, v0
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB19_3
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
-; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s4
; GFX9-NEXT: .LBB19_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -12796,27 +12376,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
;
; GFX1064-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1064: ; %bb.0:
-; GFX1064-NEXT: s_mov_b64 s[2:3], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-NEXT: v_readfirstlane_b32 s6, v0
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB19_3
; GFX1064-NEXT: ; %bb.1:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
-; GFX1064-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX1064-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v1, s4
+; GFX1064-NEXT: v_mov_b32_e32 v1, s2
+; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: .LBB19_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -12828,31 +12409,32 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
;
; GFX1032-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: s_mov_b32 s3, exec_lo
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s3, 0
+; GFX1032-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB19_3
; GFX1032-NEXT: ; %bb.1:
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
-; GFX1032-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-NEXT: .LBB19_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB19_2
; GFX1032-NEXT: .LBB19_3:
; GFX1032-NEXT: s_endpgm
@@ -12860,74 +12442,78 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
; GFX1164-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1164: ; %bb.0:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
; GFX1164-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1164-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB19_2
; GFX1164-NEXT: ; %bb.1:
; GFX1164-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164-NEXT: v_cvt_f32_ubyte0_e32 v0, s0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-NEXT: v_mov_b32_e32 v1, s0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_atomic_add_f32 v1, v0, s[2:3]
+; GFX1164-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GFX1164-NEXT: .LBB19_2:
; GFX1164-NEXT: s_endpgm
;
; GFX1132-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1132: ; %bb.0:
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
; GFX1132-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1132-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB19_2
; GFX1132-NEXT: ; %bb.1:
; GFX1132-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_cvt_f32_ubyte0_e32 v0, s0
-; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mul_f32 v0, 4.0, v0
+; GFX1132-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_atomic_add_f32 v1, v0, s[2:3]
+; GFX1132-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GFX1132-NEXT: .LBB19_2:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB19_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
-; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, s6
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, s2
+; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
; GFX7LESS-DPP-NEXT: .LBB19_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-DPP-NEXT: v_add_f32_e32 v0, s6, v1
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB19_2
; GFX7LESS-DPP-NEXT: .LBB19_3:
@@ -12935,27 +12521,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
;
; GFX9-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s6, v0
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB19_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
-; GFX9-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX9-DPP-NEXT: .LBB19_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -12967,27 +12554,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
;
; GFX1064-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1064-DPP: ; %bb.0:
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s6, v0
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB19_3
; GFX1064-DPP-NEXT: ; %bb.1:
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX1064-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX1064-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s4
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s2
+; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: .LBB19_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -12999,31 +12587,32 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
;
; GFX1032-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: s_mov_b32 s3, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
+; GFX1032-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s2, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB19_3
; GFX1032-DPP-NEXT: ; %bb.1:
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
-; GFX1032-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-DPP-NEXT: .LBB19_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB19_2
; GFX1032-DPP-NEXT: .LBB19_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -13031,40 +12620,43 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
; GFX1164-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1164-DPP: ; %bb.0:
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1164-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB19_2
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_atomic_add_f32 v1, v0, s[2:3]
+; GFX1164-DPP-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GFX1164-DPP-NEXT: .LBB19_2:
; GFX1164-DPP-NEXT: s_endpgm
;
; GFX1132-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1132-DPP: ; %bb.0:
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
; GFX1132-DPP-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1132-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB19_2
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s0
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mul_f32 v0, 4.0, v0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_atomic_add_f32 v1, v0, s[2:3]
+; GFX1132-DPP-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GFX1132-DPP-NEXT: .LBB19_2:
; GFX1132-DPP-NEXT: s_endpgm
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 monotonic, align 4, !amdgpu.no.fine.grained.memory !1, !amdgpu.no.remote.memory !1
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
index c1e5a18ec6aa2..f024ebc867271 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
@@ -43,7 +43,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_agent_scope_
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -51,8 +52,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_agent_scope_
; GFX9-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-NEXT: v_max_f32_e32 v0, 4.0, v0
-; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -152,7 +153,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_agent_scope_
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-DPP-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -160,8 +162,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_agent_scope_
; GFX9-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-DPP-NEXT: v_max_f32_e32 v0, 4.0, v0
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -269,33 +271,30 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX7LESS-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s2
-; GFX7LESS-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; GFX7LESS-NEXT: v_mul_f32_e64 v2, 1.0, s4
-; GFX7LESS-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB1_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b32 s2, 0x7fc00000
+; GFX7LESS-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s3, s[0:1]
+; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s3
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s4
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX7LESS-NEXT: v_max_f32_e32 v2, s2, v2
+; GFX7LESS-NEXT: s_or_b64 s[4:5], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s2, v2
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB1_4
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: buffer_atomic_fmax v1, off, s[0:3], 0
+; GFX7LESS-NEXT: buffer_atomic_fmax v0, off, s[0:3], 0
; GFX7LESS-NEXT: .LBB1_4:
; GFX7LESS-NEXT: s_endpgm
;
@@ -329,38 +328,36 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x7fc00000
-; GFX9-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_mov_b32 s4, 0x7fc00000
+; GFX9-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s3, v0, s2
-; GFX9-NEXT: v_max_f32_e64 v1, s3, s3
-; GFX9-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX9-NEXT: v_max_f32_e32 v2, v2, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX9-NEXT: v_max_f32_e32 v2, s4, v2
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9-NEXT: s_cbranch_execz .LBB1_5
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX9-NEXT: v_max_f32_e64 v3, s4, s4
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX9-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-NEXT: .LBB1_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-NEXT: v_max_f32_e32 v0, v0, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_max_f32_e32 v0, v0, v3
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -400,30 +397,28 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v1, 0x7fc00000
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX1064-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1064-NEXT: v_max_f32_e64 v2, s3, s3
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX1064-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1064-NEXT: s_mov_b32 s2, 0x7fc00000
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s3, s[0:1]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1064-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1064-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1064-NEXT: v_max_f32_e64 v2, s2, s4
+; GFX1064-NEXT: v_readfirstlane_b32 s2, v2
; GFX1064-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064-NEXT: s_cbranch_execz .LBB1_4
; GFX1064-NEXT: ; %bb.3:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-NEXT: v_mov_b32_e32 v0, s2
+; GFX1064-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_atomic_fmax v0, v1, s[0:1]
+; GFX1064-NEXT: global_atomic_fmax v1, v0, s[0:1]
; GFX1064-NEXT: .LBB1_4:
; GFX1064-NEXT: s_endpgm
;
@@ -457,29 +452,27 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_max_f32_e64 v2, s2, s2
-; GFX1032-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s1, exec_lo
+; GFX1032-NEXT: s_mov_b32 s0, 0x7fc00000
+; GFX1032-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s2, s1
+; GFX1032-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1032-NEXT: s_bitset0_b32 s1, s2
+; GFX1032-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1032-NEXT: v_max_f32_e64 v2, s0, s3
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v2
; GFX1032-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB1_4
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v0, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_atomic_fmax v0, v1, s[0:1]
+; GFX1032-NEXT: global_atomic_fmax v1, v0, s[2:3]
; GFX1032-NEXT: .LBB1_4:
; GFX1032-NEXT: s_endpgm
;
@@ -503,34 +496,31 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0x7fc00000
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
-; GFX1164-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_max_f32_e64 v2, s3, s3
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: s_mov_b32 s2, 0x7fc00000
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1164-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: v_max_f32_e64 v2, s2, s4
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_readfirstlane_b32 s2, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: ; %bb.2:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB1_4
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-NEXT: v_mov_b32_e32 v0, s2
+; GFX1164-NEXT: v_mov_b32_e32 v1, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_atomic_max_f32 v0, v1, s[0:1]
+; GFX1164-NEXT: global_atomic_max_f32 v1, v0, s[0:1]
; GFX1164-NEXT: .LBB1_4:
; GFX1164-NEXT: s_endpgm
;
@@ -555,32 +545,28 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_max_f32_e64 v2, s2, s2
-; GFX1132-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1132-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: s_mov_b32 s0, 0x7fc00000
+; GFX1132-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1132-NEXT: s_bitset0_b32 s1, s2
+; GFX1132-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132-NEXT: v_max_f32_e64 v2, s0, s3
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1132-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB1_4
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v0, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_atomic_max_f32 v0, v1, s[0:1]
+; GFX1132-NEXT: global_atomic_max_f32 v1, v0, s[2:3]
; GFX1132-NEXT: .LBB1_4:
; GFX1132-NEXT: s_endpgm
;
@@ -653,37 +639,19 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[0:1]
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v5
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v4 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX9-DPP-NEXT: v_max_f32_e32 v3, v4, v3
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
@@ -693,14 +661,14 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_max_f32_e64 v6, s4, s4
+; GFX9-DPP-NEXT: v_max_f32_e64 v4, s4, s4
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-DPP-NEXT: .LBB1_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-DPP-NEXT: v_max_f32_e32 v0, v0, v6
+; GFX9-DPP-NEXT: v_max_f32_e32 v0, v0, v4
; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -742,46 +710,38 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7fc00000, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1064-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1064-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v4, v4
+; GFX1064-DPP-NEXT: ds_permute_b32 v4, v5, v3
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v3, 0
-; GFX1064-DPP-NEXT: v_max_f32_e64 v3, s2, s2
-; GFX1064-DPP-NEXT: v_max_f32_e64 v4, s3, s3
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v4, v3
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b32 s0, s2
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_atomic_fmax v1, v0, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_atomic_fmax v1, v0, s[2:3]
; GFX1064-DPP-NEXT: .LBB1_2:
; GFX1064-DPP-NEXT: s_endpgm
;
@@ -816,38 +776,28 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7fc00000, v0, s0
+; GFX1032-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v3, 31
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s0, s1
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_atomic_fmax v1, v0, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_atomic_fmax v1, v0, s[2:3]
; GFX1032-DPP-NEXT: .LBB1_2:
; GFX1032-DPP-NEXT: s_endpgm
;
@@ -872,56 +822,44 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s[0:1]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v2, v1
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v1, 0x7fc00000, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: v_permlane64_b32 v2, v1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b32 s0, s2
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_atomic_max_f32 v4, v0, s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_max_f32 v4, v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB1_2:
; GFX1164-DPP-NEXT: s_endpgm
;
@@ -947,45 +885,30 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v1, 0x7fc00000, v0, s0
+; GFX1132-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v2, v2 :: v_dual_max_f32 v1, v1, v1
-; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v2, v1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v3, v3 :: v_dual_mov_b32 v3, 0x7fc00000
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v3, v3 :: v_dual_mov_b32 v3, 0x7fc00000
-; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX1132-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v1, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s0, s1
+; GFX1132-DPP-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_atomic_max_f32 v4, v0, s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_max_f32 v3, v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB1_2:
; GFX1132-DPP-NEXT: s_endpgm
%divValue = call float @div.float.value()
@@ -1021,7 +944,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_one_as_scope
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -1029,8 +953,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_one_as_scope
; GFX9-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-NEXT: v_max_f32_e32 v0, 4.0, v0
-; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -1130,7 +1054,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_one_as_scope
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-DPP-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -1138,8 +1063,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_one_as_scope
; GFX9-DPP-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-DPP-NEXT: v_max_f32_e32 v0, 4.0, v0
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -1248,33 +1173,30 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX7LESS-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s2
-; GFX7LESS-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; GFX7LESS-NEXT: v_mul_f32_e64 v2, 1.0, s4
-; GFX7LESS-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB3_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b32 s2, 0x7fc00000
+; GFX7LESS-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s3, s[0:1]
+; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s3
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s4
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX7LESS-NEXT: v_max_f32_e32 v2, s2, v2
+; GFX7LESS-NEXT: s_or_b64 s[4:5], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s2, v2
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB3_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB3_4
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: buffer_atomic_fmax v1, off, s[0:3], 0
+; GFX7LESS-NEXT: buffer_atomic_fmax v0, off, s[0:3], 0
; GFX7LESS-NEXT: .LBB3_4:
; GFX7LESS-NEXT: s_endpgm
;
@@ -1308,38 +1230,36 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x7fc00000
-; GFX9-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_mov_b32 s4, 0x7fc00000
+; GFX9-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s3, v0, s2
-; GFX9-NEXT: v_max_f32_e64 v1, s3, s3
-; GFX9-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX9-NEXT: v_max_f32_e32 v2, v2, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX9-NEXT: v_max_f32_e32 v2, s4, v2
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9-NEXT: s_cbranch_execz .LBB3_5
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX9-NEXT: v_max_f32_e64 v3, s4, s4
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX9-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-NEXT: .LBB3_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-NEXT: v_max_f32_e32 v0, v0, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_max_f32_e32 v0, v0, v3
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -1379,30 +1299,28 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v1, 0x7fc00000
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX1064-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1064-NEXT: v_max_f32_e64 v2, s3, s3
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX1064-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1064-NEXT: s_mov_b32 s2, 0x7fc00000
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s3, s[0:1]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1064-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1064-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1064-NEXT: v_max_f32_e64 v2, s2, s4
+; GFX1064-NEXT: v_readfirstlane_b32 s2, v2
; GFX1064-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064-NEXT: s_cbranch_execz .LBB3_4
; GFX1064-NEXT: ; %bb.3:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-NEXT: v_mov_b32_e32 v0, s2
+; GFX1064-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_atomic_fmax v0, v1, s[0:1]
+; GFX1064-NEXT: global_atomic_fmax v1, v0, s[0:1]
; GFX1064-NEXT: .LBB3_4:
; GFX1064-NEXT: s_endpgm
;
@@ -1436,29 +1354,27 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_max_f32_e64 v2, s2, s2
-; GFX1032-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s1, exec_lo
+; GFX1032-NEXT: s_mov_b32 s0, 0x7fc00000
+; GFX1032-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s2, s1
+; GFX1032-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1032-NEXT: s_bitset0_b32 s1, s2
+; GFX1032-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1032-NEXT: v_max_f32_e64 v2, s0, s3
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v2
; GFX1032-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB3_4
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v0, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_atomic_fmax v0, v1, s[0:1]
+; GFX1032-NEXT: global_atomic_fmax v1, v0, s[2:3]
; GFX1032-NEXT: .LBB3_4:
; GFX1032-NEXT: s_endpgm
;
@@ -1482,34 +1398,31 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0x7fc00000
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
-; GFX1164-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_max_f32_e64 v2, s3, s3
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: s_mov_b32 s2, 0x7fc00000
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1164-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: v_max_f32_e64 v2, s2, s4
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_readfirstlane_b32 s2, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: ; %bb.2:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB3_4
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-NEXT: v_mov_b32_e32 v0, s2
+; GFX1164-NEXT: v_mov_b32_e32 v1, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_atomic_max_f32 v0, v1, s[0:1]
+; GFX1164-NEXT: global_atomic_max_f32 v1, v0, s[0:1]
; GFX1164-NEXT: .LBB3_4:
; GFX1164-NEXT: s_endpgm
;
@@ -1534,32 +1447,28 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_max_f32_e64 v2, s2, s2
-; GFX1132-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1132-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: s_mov_b32 s0, 0x7fc00000
+; GFX1132-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1132-NEXT: s_bitset0_b32 s1, s2
+; GFX1132-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132-NEXT: v_max_f32_e64 v2, s0, s3
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1132-NEXT: s_cbranch_scc1 .LBB3_1
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB3_4
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v0, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_atomic_max_f32 v0, v1, s[0:1]
+; GFX1132-NEXT: global_atomic_max_f32 v1, v0, s[2:3]
; GFX1132-NEXT: .LBB3_4:
; GFX1132-NEXT: s_endpgm
;
@@ -1632,37 +1541,19 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[0:1]
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v5
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v4 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX9-DPP-NEXT: v_max_f32_e32 v3, v4, v3
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
@@ -1672,14 +1563,14 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_max_f32_e64 v6, s4, s4
+; GFX9-DPP-NEXT: v_max_f32_e64 v4, s4, s4
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-DPP-NEXT: .LBB3_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-DPP-NEXT: v_max_f32_e32 v0, v0, v6
+; GFX9-DPP-NEXT: v_max_f32_e32 v0, v0, v4
; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -1721,46 +1612,38 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7fc00000, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1064-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1064-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v4, v4
+; GFX1064-DPP-NEXT: ds_permute_b32 v4, v5, v3
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v3, 0
-; GFX1064-DPP-NEXT: v_max_f32_e64 v3, s2, s2
-; GFX1064-DPP-NEXT: v_max_f32_e64 v4, s3, s3
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v4, v3
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b32 s0, s2
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_atomic_fmax v1, v0, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_atomic_fmax v1, v0, s[2:3]
; GFX1064-DPP-NEXT: .LBB3_2:
; GFX1064-DPP-NEXT: s_endpgm
;
@@ -1795,38 +1678,28 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7fc00000, v0, s0
+; GFX1032-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v3, 31
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s0, s1
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_atomic_fmax v1, v0, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_atomic_fmax v1, v0, s[2:3]
; GFX1032-DPP-NEXT: .LBB3_2:
; GFX1032-DPP-NEXT: s_endpgm
;
@@ -1851,56 +1724,44 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s[0:1]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v2, v1
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v1, 0x7fc00000, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: v_permlane64_b32 v2, v1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b32 s0, s2
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_atomic_max_f32 v4, v0, s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_max_f32 v4, v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB3_2:
; GFX1164-DPP-NEXT: s_endpgm
;
@@ -1926,45 +1787,30 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v2, v2 :: v_dual_max_f32 v1, v1, v1
-; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v2, v1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v3, v3 :: v_dual_mov_b32 v3, 0x7fc00000
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v1, 0x7fc00000, v0, s0
+; GFX1132-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v3, v3 :: v_dual_mov_b32 v3, 0x7fc00000
-; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v1, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s0, s1
+; GFX1132-DPP-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_atomic_max_f32 v4, v0, s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_max_f32 v3, v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB3_2:
; GFX1132-DPP-NEXT: s_endpgm
%divValue = call float @div.float.value()
@@ -2001,7 +1847,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_default_scop
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -2009,8 +1856,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_default_scop
; GFX9-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-NEXT: v_max_f32_e32 v0, 4.0, v0
-; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -2110,7 +1957,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_default_scop
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-DPP-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -2118,8 +1966,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_default_scop
; GFX9-DPP-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-DPP-NEXT: v_max_f32_e32 v0, 4.0, v0
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -2227,33 +2075,30 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX7LESS-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s2
-; GFX7LESS-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; GFX7LESS-NEXT: v_mul_f32_e64 v2, 1.0, s4
-; GFX7LESS-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB5_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b32 s2, 0x7fc00000
+; GFX7LESS-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s3, s[0:1]
+; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s3
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s4
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX7LESS-NEXT: v_max_f32_e32 v2, s2, v2
+; GFX7LESS-NEXT: s_or_b64 s[4:5], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s2, v2
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB5_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB5_4
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: buffer_atomic_fmax v1, off, s[0:3], 0
+; GFX7LESS-NEXT: buffer_atomic_fmax v0, off, s[0:3], 0
; GFX7LESS-NEXT: .LBB5_4:
; GFX7LESS-NEXT: s_endpgm
;
@@ -2287,38 +2132,36 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x7fc00000
-; GFX9-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_mov_b32 s4, 0x7fc00000
+; GFX9-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s3, v0, s2
-; GFX9-NEXT: v_max_f32_e64 v1, s3, s3
-; GFX9-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX9-NEXT: v_max_f32_e32 v2, v2, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX9-NEXT: v_max_f32_e32 v2, s4, v2
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9-NEXT: s_cbranch_execz .LBB5_5
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX9-NEXT: v_max_f32_e64 v3, s4, s4
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX9-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-NEXT: .LBB5_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-NEXT: v_max_f32_e32 v0, v0, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_max_f32_e32 v0, v0, v3
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -2358,30 +2201,28 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v1, 0x7fc00000
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX1064-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1064-NEXT: v_max_f32_e64 v2, s3, s3
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX1064-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1064-NEXT: s_mov_b32 s2, 0x7fc00000
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s3, s[0:1]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1064-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1064-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1064-NEXT: v_max_f32_e64 v2, s2, s4
+; GFX1064-NEXT: v_readfirstlane_b32 s2, v2
; GFX1064-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064-NEXT: s_cbranch_execz .LBB5_4
; GFX1064-NEXT: ; %bb.3:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-NEXT: v_mov_b32_e32 v0, s2
+; GFX1064-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_atomic_fmax v0, v1, s[0:1]
+; GFX1064-NEXT: global_atomic_fmax v1, v0, s[0:1]
; GFX1064-NEXT: .LBB5_4:
; GFX1064-NEXT: s_endpgm
;
@@ -2415,29 +2256,27 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_max_f32_e64 v2, s2, s2
-; GFX1032-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s1, exec_lo
+; GFX1032-NEXT: s_mov_b32 s0, 0x7fc00000
+; GFX1032-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s2, s1
+; GFX1032-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1032-NEXT: s_bitset0_b32 s1, s2
+; GFX1032-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1032-NEXT: v_max_f32_e64 v2, s0, s3
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v2
; GFX1032-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB5_4
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v0, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_atomic_fmax v0, v1, s[0:1]
+; GFX1032-NEXT: global_atomic_fmax v1, v0, s[2:3]
; GFX1032-NEXT: .LBB5_4:
; GFX1032-NEXT: s_endpgm
;
@@ -2461,34 +2300,31 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0x7fc00000
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
-; GFX1164-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_max_f32_e64 v2, s3, s3
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: s_mov_b32 s2, 0x7fc00000
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1164-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: v_max_f32_e64 v2, s2, s4
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_readfirstlane_b32 s2, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: ; %bb.2:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB5_4
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-NEXT: v_mov_b32_e32 v0, s2
+; GFX1164-NEXT: v_mov_b32_e32 v1, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_atomic_max_f32 v0, v1, s[0:1]
+; GFX1164-NEXT: global_atomic_max_f32 v1, v0, s[0:1]
; GFX1164-NEXT: .LBB5_4:
; GFX1164-NEXT: s_endpgm
;
@@ -2513,32 +2349,28 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_max_f32_e64 v2, s2, s2
-; GFX1132-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1132-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: s_mov_b32 s0, 0x7fc00000
+; GFX1132-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1132-NEXT: s_bitset0_b32 s1, s2
+; GFX1132-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132-NEXT: v_max_f32_e64 v2, s0, s3
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1132-NEXT: s_cbranch_scc1 .LBB5_1
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB5_4
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v0, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_atomic_max_f32 v0, v1, s[0:1]
+; GFX1132-NEXT: global_atomic_max_f32 v1, v0, s[2:3]
; GFX1132-NEXT: .LBB5_4:
; GFX1132-NEXT: s_endpgm
;
@@ -2611,37 +2443,19 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[0:1]
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v5
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v4 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX9-DPP-NEXT: v_max_f32_e32 v3, v4, v3
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
@@ -2651,14 +2465,14 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_max_f32_e64 v6, s4, s4
+; GFX9-DPP-NEXT: v_max_f32_e64 v4, s4, s4
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-DPP-NEXT: .LBB5_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-DPP-NEXT: v_max_f32_e32 v0, v0, v6
+; GFX9-DPP-NEXT: v_max_f32_e32 v0, v0, v4
; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -2700,46 +2514,38 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7fc00000, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1064-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1064-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v4, v4
+; GFX1064-DPP-NEXT: ds_permute_b32 v4, v5, v3
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v3, 0
-; GFX1064-DPP-NEXT: v_max_f32_e64 v3, s2, s2
-; GFX1064-DPP-NEXT: v_max_f32_e64 v4, s3, s3
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v4, v3
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b32 s0, s2
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_atomic_fmax v1, v0, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_atomic_fmax v1, v0, s[2:3]
; GFX1064-DPP-NEXT: .LBB5_2:
; GFX1064-DPP-NEXT: s_endpgm
;
@@ -2774,38 +2580,28 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7fc00000, v0, s0
+; GFX1032-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v3, 31
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s0, s1
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_atomic_fmax v1, v0, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_atomic_fmax v1, v0, s[2:3]
; GFX1032-DPP-NEXT: .LBB5_2:
; GFX1032-DPP-NEXT: s_endpgm
;
@@ -2830,56 +2626,44 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s[0:1]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v2, v1
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v1, 0x7fc00000, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: v_permlane64_b32 v2, v1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b32 s0, s2
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_atomic_max_f32 v4, v0, s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_max_f32 v4, v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB5_2:
; GFX1164-DPP-NEXT: s_endpgm
;
@@ -2905,45 +2689,30 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v1, 0x7fc00000, v0, s0
+; GFX1132-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v2, v2 :: v_dual_max_f32 v1, v1, v1
-; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v2, v1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v3, v3 :: v_dual_mov_b32 v3, 0x7fc00000
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v3, v3 :: v_dual_mov_b32 v3, 0x7fc00000
-; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX1132-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v1, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s0, s1
+; GFX1132-DPP-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_atomic_max_f32 v4, v0, s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_max_f32 v3, v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB5_2:
; GFX1132-DPP-NEXT: s_endpgm
%divValue = call float @div.float.value()
@@ -2979,8 +2748,9 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
; GFX9-NEXT: s_cbranch_execz .LBB6_3
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_max_f64 v[4:5], 4.0, 4.0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: v_mov_b32_e32 v6, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -2989,8 +2759,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
; GFX9-NEXT: .LBB6_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
@@ -3044,7 +2814,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
; GFX1164-NEXT: s_cbranch_execz .LBB6_3
; GFX1164-NEXT: ; %bb.1:
; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1164-NEXT: v_mov_b32_e32 v6, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
@@ -3055,8 +2826,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1164-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
@@ -3078,7 +2849,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
; GFX1132-NEXT: s_cbranch_execz .LBB6_3
; GFX1132-NEXT: ; %bb.1:
; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1132-NEXT: v_mov_b32_e32 v6, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
@@ -3087,8 +2859,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1132-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
@@ -3126,8 +2898,9 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
; GFX9-DPP-NEXT: s_cbranch_execz .LBB6_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -3136,8 +2909,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
; GFX9-DPP-NEXT: .LBB6_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1
@@ -3191,7 +2964,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB6_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -3202,8 +2976,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1
@@ -3225,7 +2999,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB6_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -3234,8 +3009,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
@@ -3281,35 +3056,35 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX7LESS-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s4
-; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s4
-; GFX7LESS-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX7LESS-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX7LESS-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB7_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: s_mov_b32 s0, 0
+; GFX7LESS-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
+; GFX7LESS-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s6
+; GFX7LESS-NEXT: v_readlane_b32 s5, v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, s1
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s0
+; GFX7LESS-NEXT: v_max_f64 v[3:4], s[4:5], v[3:4]
+; GFX7LESS-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX7LESS-NEXT: v_readfirstlane_b32 s0, v3
+; GFX7LESS-NEXT: s_or_b64 s[4:5], s[2:3], s[2:3]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s1, v4
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB7_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB7_4
; GFX7LESS-NEXT: ; %bb.3:
-; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
-; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[34:35], 0x9
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s0
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s1
+; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT: s_mov_b32 s6, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: buffer_atomic_fmax_x2 v[2:3], off, s[0:3], 0
+; GFX7LESS-NEXT: buffer_atomic_fmax_x2 v[0:1], off, s[4:7], 0
; GFX7LESS-NEXT: .LBB7_4:
; GFX7LESS-NEXT: s_endpgm
;
@@ -3343,46 +3118,46 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX9-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s3, v1, s4
-; GFX9-NEXT: v_readlane_b32 s2, v0, s4
-; GFX9-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX9-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX9-NEXT: v_max_f64 v[4:5], v[2:3], v[4:5]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: s_mov_b32 s0, 0
+; GFX9-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX9-NEXT: s_mov_b64 s[2:3], exec
+; GFX9-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9-NEXT: v_readlane_b32 s4, v0, s6
+; GFX9-NEXT: v_readlane_b32 s5, v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_max_f64 v[3:4], s[4:5], v[3:4]
+; GFX9-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v3
+; GFX9-NEXT: v_readfirstlane_b32 s1, v4
; GFX9-NEXT: s_cbranch_scc1 .LBB7_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB7_5
; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX9-NEXT: s_mov_b64 s[0:1], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[2:3]
; GFX9-NEXT: .LBB7_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_cbranch_execnz .LBB7_4
; GFX9-NEXT: .LBB7_5:
; GFX9-NEXT: s_endpgm
@@ -3417,32 +3192,32 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX1064-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1064-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1064-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX1064-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_mov_b32 s0, 0
+; GFX1064-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1064-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1064-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1064-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064-NEXT: v_max_f64 v[3:4], s[4:5], s[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v4
; GFX1064-NEXT: s_cbranch_scc1 .LBB7_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB7_4
; GFX1064-NEXT: ; %bb.3:
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v0, s0
+; GFX1064-NEXT: v_mov_b32_e32 v1, s1
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_atomic_fmax_x2 v0, v[2:3], s[0:1]
+; GFX1064-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[2:3]
; GFX1064-NEXT: .LBB7_4:
; GFX1064-NEXT: s_endpgm
;
@@ -3476,31 +3251,31 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1032-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1032-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, 0
+; GFX1032-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1032-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1032-NEXT: s_bitset0_b32 s2, s3
+; GFX1032-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032-NEXT: v_max_f64 v[3:4], s[4:5], s[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v4
; GFX1032-NEXT: s_cbranch_scc1 .LBB7_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB7_4
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v0, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_atomic_fmax_x2 v0, v[2:3], s[0:1]
+; GFX1032-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[2:3]
; GFX1032-NEXT: .LBB7_4:
; GFX1032-NEXT: s_endpgm
;
@@ -3524,52 +3299,49 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX1164-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1164-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_mov_b32 s0, 0
+; GFX1164-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_max_f64 v[4:5], v[2:3], v[4:5]
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s6, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1164-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1164-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164-NEXT: v_max_f64 v[3:4], s[4:5], s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v4
; GFX1164-NEXT: s_cbranch_scc1 .LBB7_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: ; %bb.2:
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB7_5
; GFX1164-NEXT: ; %bb.3:
-; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[0:1], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[2:3]
; GFX1164-NEXT: .LBB7_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1]
; GFX1164-NEXT: s_cbranch_execnz .LBB7_4
; GFX1164-NEXT: .LBB7_5:
; GFX1164-NEXT: s_endpgm
@@ -3595,49 +3367,47 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 0x7ff80000
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX1132-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s0, 0
+; GFX1132-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1132-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1132-NEXT: s_bitset0_b32 s2, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1132-NEXT: v_max_f64 v[4:5], v[2:3], v[4:5]
+; GFX1132-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132-NEXT: v_max_f64 v[3:4], s[4:5], s[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v4
; GFX1132-NEXT: s_cbranch_scc1 .LBB7_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB7_5
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
; GFX1132-NEXT: v_mov_b32_e32 v6, 0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX1132-NEXT: s_mov_b32 s0, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[2:3]
; GFX1132-NEXT: .LBB7_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX1132-NEXT: s_cbranch_execnz .LBB7_4
; GFX1132-NEXT: .LBB7_5:
; GFX1132-NEXT: s_endpgm
@@ -3708,80 +3478,74 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1]
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0x7ff80000
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, v3, v1, s[0:1]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v7 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[4:5], v[6:7]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
-; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
+; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3]
+; GFX9-DPP-NEXT: global_load_dwordx2 v[10:11], v2, s[0:1]
; GFX9-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_max_f64 v[1:2], s[0:1], s[0:1]
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_max_f64 v[9:10], v[11:12], v[11:12]
-; GFX9-DPP-NEXT: v_max_f64 v[9:10], v[9:10], v[1:2]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[10:11], v[10:11]
+; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[8:9], v2, v[8:11], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v2
-; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v1
-; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[10:11]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v9
+; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v10, v8
+; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB7_2
; GFX9-DPP-NEXT: .LBB7_3:
; GFX9-DPP-NEXT: s_endpgm
@@ -3817,57 +3581,59 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x7ff80000, v1, s[0:1]
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7ff80000, v1, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s5, v4, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], s[4:5], s[4:5]
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], s[2:3], s[2:3]
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[5:6], v[3:4]
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v7
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: ds_permute_b32 v6, v5, v3
+; GFX1064-DPP-NEXT: ds_permute_b32 v7, v5, v4
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[6:7]
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 63
+; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB7_2
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s1
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[2:3]
; GFX1064-DPP-NEXT: .LBB7_2:
; GFX1064-DPP-NEXT: s_endpgm
;
@@ -3901,50 +3667,48 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x7ff80000, v1, s0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[5:6], v[3:4]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s2
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7ff80000, v1, s2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT: v_readlane_b32 s0, v3, 31
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v4, 31
+; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB7_2
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[2:3]
; GFX1032-DPP-NEXT: .LBB7_2:
; GFX1032-DPP-NEXT: s_endpgm
;
@@ -3969,82 +3733,85 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s[0:1]
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v1, s[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[4:5], v[2:3]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v5, v3
-; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, -1, v4
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v4, 32, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v4, 4, v4
+; GFX1164-DPP-NEXT: ds_permute_b32 v5, v4, v2
+; GFX1164-DPP-NEXT: ds_permute_b32 v6, v4, v3
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[5:6]
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT: v_readlane_b32 s3, v3, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v11, 0
+; GFX1164-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b64 v[9:10], v11, s[0:1]
; GFX1164-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[8:9], v[8:9]
+; GFX1164-DPP-NEXT: v_max_f64 v[7:8], v[9:10], v[9:10]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[6:7], v[0:1]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[7:8], v11, v[7:10], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8
; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6
; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
@@ -4074,56 +3841,51 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, 0x7ff80000 :: v_dual_mov_b32 v2, 0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[4:5], v[2:3]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s0
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v1, s0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_readlane_b32 s2, v2, 31
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_readlane_b32 s3, v3, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s4, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v10, 0
+; GFX1132-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
; GFX1132-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
@@ -4136,9 +3898,9 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB7_2
; GFX1132-DPP-NEXT: .LBB7_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -4175,8 +3937,9 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_a
; GFX9-NEXT: s_cbranch_execz .LBB8_3
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_max_f64 v[4:5], 4.0, 4.0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: v_mov_b32_e32 v6, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -4185,8 +3948,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_a
; GFX9-NEXT: .LBB8_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
@@ -4240,7 +4003,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_a
; GFX1164-NEXT: s_cbranch_execz .LBB8_3
; GFX1164-NEXT: ; %bb.1:
; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1164-NEXT: v_mov_b32_e32 v6, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
@@ -4251,8 +4015,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_a
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1164-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
@@ -4274,7 +4038,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_a
; GFX1132-NEXT: s_cbranch_execz .LBB8_3
; GFX1132-NEXT: ; %bb.1:
; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1132-NEXT: v_mov_b32_e32 v6, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
@@ -4283,8 +4048,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_a
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1132-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
@@ -4322,8 +4087,9 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_a
; GFX9-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -4332,8 +4098,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_a
; GFX9-DPP-NEXT: .LBB8_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1
@@ -4387,7 +4153,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_a
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -4398,8 +4165,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_a
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1
@@ -4421,7 +4188,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_a
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -4430,8 +4198,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_a
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
@@ -4477,35 +4245,35 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX7LESS-NEXT: .LBB9_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s4
-; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s4
-; GFX7LESS-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX7LESS-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX7LESS-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB9_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: s_mov_b32 s0, 0
+; GFX7LESS-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
+; GFX7LESS-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s6
+; GFX7LESS-NEXT: v_readlane_b32 s5, v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, s1
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s0
+; GFX7LESS-NEXT: v_max_f64 v[3:4], s[4:5], v[3:4]
+; GFX7LESS-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX7LESS-NEXT: v_readfirstlane_b32 s0, v3
+; GFX7LESS-NEXT: s_or_b64 s[4:5], s[2:3], s[2:3]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s1, v4
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB9_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB9_4
; GFX7LESS-NEXT: ; %bb.3:
-; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
-; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[34:35], 0x9
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s0
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s1
+; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT: s_mov_b32 s6, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: buffer_atomic_fmax_x2 v[2:3], off, s[0:3], 0
+; GFX7LESS-NEXT: buffer_atomic_fmax_x2 v[0:1], off, s[4:7], 0
; GFX7LESS-NEXT: .LBB9_4:
; GFX7LESS-NEXT: s_endpgm
;
@@ -4539,46 +4307,46 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX9-NEXT: .LBB9_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s3, v1, s4
-; GFX9-NEXT: v_readlane_b32 s2, v0, s4
-; GFX9-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX9-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX9-NEXT: v_max_f64 v[4:5], v[2:3], v[4:5]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: s_mov_b32 s0, 0
+; GFX9-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX9-NEXT: s_mov_b64 s[2:3], exec
+; GFX9-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9-NEXT: v_readlane_b32 s4, v0, s6
+; GFX9-NEXT: v_readlane_b32 s5, v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_max_f64 v[3:4], s[4:5], v[3:4]
+; GFX9-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v3
+; GFX9-NEXT: v_readfirstlane_b32 s1, v4
; GFX9-NEXT: s_cbranch_scc1 .LBB9_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB9_5
; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX9-NEXT: s_mov_b64 s[0:1], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[2:3]
; GFX9-NEXT: .LBB9_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_cbranch_execnz .LBB9_4
; GFX9-NEXT: .LBB9_5:
; GFX9-NEXT: s_endpgm
@@ -4613,32 +4381,32 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB9_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX1064-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1064-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1064-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX1064-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_mov_b32 s0, 0
+; GFX1064-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1064-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1064-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1064-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064-NEXT: v_max_f64 v[3:4], s[4:5], s[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v4
; GFX1064-NEXT: s_cbranch_scc1 .LBB9_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB9_4
; GFX1064-NEXT: ; %bb.3:
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v0, s0
+; GFX1064-NEXT: v_mov_b32_e32 v1, s1
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_atomic_fmax_x2 v0, v[2:3], s[0:1]
+; GFX1064-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[2:3]
; GFX1064-NEXT: .LBB9_4:
; GFX1064-NEXT: s_endpgm
;
@@ -4672,31 +4440,31 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB9_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1032-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1032-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, 0
+; GFX1032-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1032-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1032-NEXT: s_bitset0_b32 s2, s3
+; GFX1032-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032-NEXT: v_max_f64 v[3:4], s[4:5], s[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v4
; GFX1032-NEXT: s_cbranch_scc1 .LBB9_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB9_4
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v0, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_atomic_fmax_x2 v0, v[2:3], s[0:1]
+; GFX1032-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[2:3]
; GFX1032-NEXT: .LBB9_4:
; GFX1032-NEXT: s_endpgm
;
@@ -4720,52 +4488,49 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB9_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX1164-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1164-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_mov_b32 s0, 0
+; GFX1164-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_max_f64 v[4:5], v[2:3], v[4:5]
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s6, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1164-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1164-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164-NEXT: v_max_f64 v[3:4], s[4:5], s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v4
; GFX1164-NEXT: s_cbranch_scc1 .LBB9_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: ; %bb.2:
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB9_5
; GFX1164-NEXT: ; %bb.3:
-; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[0:1], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[2:3]
; GFX1164-NEXT: .LBB9_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1]
; GFX1164-NEXT: s_cbranch_execnz .LBB9_4
; GFX1164-NEXT: .LBB9_5:
; GFX1164-NEXT: s_endpgm
@@ -4791,49 +4556,47 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 0x7ff80000
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB9_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX1132-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s0, 0
+; GFX1132-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1132-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1132-NEXT: s_bitset0_b32 s2, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1132-NEXT: v_max_f64 v[4:5], v[2:3], v[4:5]
+; GFX1132-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132-NEXT: v_max_f64 v[3:4], s[4:5], s[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v4
; GFX1132-NEXT: s_cbranch_scc1 .LBB9_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB9_5
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
; GFX1132-NEXT: v_mov_b32_e32 v6, 0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX1132-NEXT: s_mov_b32 s0, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[2:3]
; GFX1132-NEXT: .LBB9_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX1132-NEXT: s_cbranch_execnz .LBB9_4
; GFX1132-NEXT: .LBB9_5:
; GFX1132-NEXT: s_endpgm
@@ -4904,80 +4667,74 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1]
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0x7ff80000
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, v3, v1, s[0:1]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v7 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[4:5], v[6:7]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
-; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
+; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3]
+; GFX9-DPP-NEXT: global_load_dwordx2 v[10:11], v2, s[0:1]
; GFX9-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_max_f64 v[1:2], s[0:1], s[0:1]
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_max_f64 v[9:10], v[11:12], v[11:12]
-; GFX9-DPP-NEXT: v_max_f64 v[9:10], v[9:10], v[1:2]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[10:11], v[10:11]
+; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[8:9], v2, v[8:11], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v2
-; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v1
-; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[10:11]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v9
+; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v10, v8
+; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB9_2
; GFX9-DPP-NEXT: .LBB9_3:
; GFX9-DPP-NEXT: s_endpgm
@@ -5013,57 +4770,59 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x7ff80000, v1, s[0:1]
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7ff80000, v1, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s5, v4, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], s[4:5], s[4:5]
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], s[2:3], s[2:3]
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[5:6], v[3:4]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v7
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: ds_permute_b32 v6, v5, v3
+; GFX1064-DPP-NEXT: ds_permute_b32 v7, v5, v4
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[6:7]
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 63
+; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB9_2
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s1
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[2:3]
; GFX1064-DPP-NEXT: .LBB9_2:
; GFX1064-DPP-NEXT: s_endpgm
;
@@ -5097,50 +4856,48 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x7ff80000, v1, s0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[5:6], v[3:4]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s2
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7ff80000, v1, s2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT: v_readlane_b32 s0, v3, 31
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v4, 31
+; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB9_2
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[2:3]
; GFX1032-DPP-NEXT: .LBB9_2:
; GFX1032-DPP-NEXT: s_endpgm
;
@@ -5165,82 +4922,85 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s[0:1]
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v1, s[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[4:5], v[2:3]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v5, v3
-; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, -1, v4
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v4, 32, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v4, 4, v4
+; GFX1164-DPP-NEXT: ds_permute_b32 v5, v4, v2
+; GFX1164-DPP-NEXT: ds_permute_b32 v6, v4, v3
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[5:6]
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT: v_readlane_b32 s3, v3, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v11, 0
+; GFX1164-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b64 v[9:10], v11, s[0:1]
; GFX1164-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[8:9], v[8:9]
+; GFX1164-DPP-NEXT: v_max_f64 v[7:8], v[9:10], v[9:10]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[6:7], v[0:1]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[7:8], v11, v[7:10], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8
; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6
; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
@@ -5270,56 +5030,51 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, 0x7ff80000 :: v_dual_mov_b32 v2, 0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[4:5], v[2:3]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s0
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v1, s0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_readlane_b32 s2, v2, 31
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_readlane_b32 s3, v3, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s4, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v10, 0
+; GFX1132-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
; GFX1132-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
@@ -5332,9 +5087,9 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB9_2
; GFX1132-DPP-NEXT: .LBB9_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -5371,8 +5126,9 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
; GFX9-NEXT: s_cbranch_execz .LBB10_3
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_max_f64 v[4:5], 4.0, 4.0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: v_mov_b32_e32 v6, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -5381,8 +5137,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
; GFX9-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
@@ -5436,7 +5192,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
; GFX1164-NEXT: s_cbranch_execz .LBB10_3
; GFX1164-NEXT: ; %bb.1:
; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1164-NEXT: v_mov_b32_e32 v6, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
@@ -5447,8 +5204,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1164-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
@@ -5470,7 +5227,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
; GFX1132-NEXT: s_cbranch_execz .LBB10_3
; GFX1132-NEXT: ; %bb.1:
; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1132-NEXT: v_mov_b32_e32 v6, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
@@ -5479,8 +5237,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1132-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
@@ -5518,8 +5276,9 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
; GFX9-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -5528,8 +5287,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
; GFX9-DPP-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1
@@ -5583,7 +5342,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -5594,8 +5354,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1
@@ -5617,7 +5377,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -5626,8 +5387,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
@@ -5673,35 +5434,35 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX7LESS-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s4
-; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s4
-; GFX7LESS-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX7LESS-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX7LESS-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB11_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: s_mov_b32 s0, 0
+; GFX7LESS-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
+; GFX7LESS-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s6
+; GFX7LESS-NEXT: v_readlane_b32 s5, v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, s1
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s0
+; GFX7LESS-NEXT: v_max_f64 v[3:4], s[4:5], v[3:4]
+; GFX7LESS-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX7LESS-NEXT: v_readfirstlane_b32 s0, v3
+; GFX7LESS-NEXT: s_or_b64 s[4:5], s[2:3], s[2:3]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s1, v4
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB11_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB11_4
; GFX7LESS-NEXT: ; %bb.3:
-; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
-; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[34:35], 0x9
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s0
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s1
+; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT: s_mov_b32 s6, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: buffer_atomic_fmax_x2 v[2:3], off, s[0:3], 0
+; GFX7LESS-NEXT: buffer_atomic_fmax_x2 v[0:1], off, s[4:7], 0
; GFX7LESS-NEXT: .LBB11_4:
; GFX7LESS-NEXT: s_endpgm
;
@@ -5735,46 +5496,46 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX9-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s3, v1, s4
-; GFX9-NEXT: v_readlane_b32 s2, v0, s4
-; GFX9-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX9-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX9-NEXT: v_max_f64 v[4:5], v[2:3], v[4:5]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: s_mov_b32 s0, 0
+; GFX9-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX9-NEXT: s_mov_b64 s[2:3], exec
+; GFX9-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9-NEXT: v_readlane_b32 s4, v0, s6
+; GFX9-NEXT: v_readlane_b32 s5, v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_max_f64 v[3:4], s[4:5], v[3:4]
+; GFX9-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v3
+; GFX9-NEXT: v_readfirstlane_b32 s1, v4
; GFX9-NEXT: s_cbranch_scc1 .LBB11_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB11_5
; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX9-NEXT: s_mov_b64 s[0:1], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[2:3]
; GFX9-NEXT: .LBB11_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_cbranch_execnz .LBB11_4
; GFX9-NEXT: .LBB11_5:
; GFX9-NEXT: s_endpgm
@@ -5809,32 +5570,32 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX1064-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1064-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1064-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX1064-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_mov_b32 s0, 0
+; GFX1064-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1064-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1064-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1064-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064-NEXT: v_max_f64 v[3:4], s[4:5], s[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v4
; GFX1064-NEXT: s_cbranch_scc1 .LBB11_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB11_4
; GFX1064-NEXT: ; %bb.3:
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v0, s0
+; GFX1064-NEXT: v_mov_b32_e32 v1, s1
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_atomic_fmax_x2 v0, v[2:3], s[0:1]
+; GFX1064-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[2:3]
; GFX1064-NEXT: .LBB11_4:
; GFX1064-NEXT: s_endpgm
;
@@ -5868,31 +5629,31 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1032-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1032-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, 0
+; GFX1032-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1032-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1032-NEXT: s_bitset0_b32 s2, s3
+; GFX1032-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032-NEXT: v_max_f64 v[3:4], s[4:5], s[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v4
; GFX1032-NEXT: s_cbranch_scc1 .LBB11_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB11_4
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v0, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_atomic_fmax_x2 v0, v[2:3], s[0:1]
+; GFX1032-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[2:3]
; GFX1032-NEXT: .LBB11_4:
; GFX1032-NEXT: s_endpgm
;
@@ -5916,52 +5677,49 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX1164-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1164-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_mov_b32 s0, 0
+; GFX1164-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_max_f64 v[4:5], v[2:3], v[4:5]
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s6, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1164-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1164-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164-NEXT: v_max_f64 v[3:4], s[4:5], s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v4
; GFX1164-NEXT: s_cbranch_scc1 .LBB11_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: ; %bb.2:
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB11_5
; GFX1164-NEXT: ; %bb.3:
-; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[0:1], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[2:3]
; GFX1164-NEXT: .LBB11_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1]
; GFX1164-NEXT: s_cbranch_execnz .LBB11_4
; GFX1164-NEXT: .LBB11_5:
; GFX1164-NEXT: s_endpgm
@@ -5987,49 +5745,47 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 0x7ff80000
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX1132-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s0, 0
+; GFX1132-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1132-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1132-NEXT: s_bitset0_b32 s2, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1132-NEXT: v_max_f64 v[4:5], v[2:3], v[4:5]
+; GFX1132-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132-NEXT: v_max_f64 v[3:4], s[4:5], s[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v4
; GFX1132-NEXT: s_cbranch_scc1 .LBB11_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB11_5
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
; GFX1132-NEXT: v_mov_b32_e32 v6, 0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX1132-NEXT: s_mov_b32 s0, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[2:3]
; GFX1132-NEXT: .LBB11_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX1132-NEXT: s_cbranch_execnz .LBB11_4
; GFX1132-NEXT: .LBB11_5:
; GFX1132-NEXT: s_endpgm
@@ -6100,80 +5856,74 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1]
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0x7ff80000
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, v3, v1, s[0:1]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v7 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[4:5], v[6:7]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
-; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
+; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3]
+; GFX9-DPP-NEXT: global_load_dwordx2 v[10:11], v2, s[0:1]
; GFX9-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_max_f64 v[1:2], s[0:1], s[0:1]
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_max_f64 v[9:10], v[11:12], v[11:12]
-; GFX9-DPP-NEXT: v_max_f64 v[9:10], v[9:10], v[1:2]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[10:11], v[10:11]
+; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[8:9], v2, v[8:11], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v2
-; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v1
-; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[10:11]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v9
+; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v10, v8
+; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB11_2
; GFX9-DPP-NEXT: .LBB11_3:
; GFX9-DPP-NEXT: s_endpgm
@@ -6209,57 +5959,59 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x7ff80000, v1, s[0:1]
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7ff80000, v1, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s5, v4, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], s[4:5], s[4:5]
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], s[2:3], s[2:3]
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[5:6], v[3:4]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v7
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: ds_permute_b32 v6, v5, v3
+; GFX1064-DPP-NEXT: ds_permute_b32 v7, v5, v4
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[6:7]
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 63
+; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB11_2
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s1
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[2:3]
; GFX1064-DPP-NEXT: .LBB11_2:
; GFX1064-DPP-NEXT: s_endpgm
;
@@ -6293,50 +6045,48 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x7ff80000, v1, s0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[5:6], v[3:4]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s2
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7ff80000, v1, s2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT: v_readlane_b32 s0, v3, 31
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v4, 31
+; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB11_2
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[2:3]
; GFX1032-DPP-NEXT: .LBB11_2:
; GFX1032-DPP-NEXT: s_endpgm
;
@@ -6361,82 +6111,85 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s[0:1]
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v1, s[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[4:5], v[2:3]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v5, v3
-; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, -1, v4
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v4, 32, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v4, 4, v4
+; GFX1164-DPP-NEXT: ds_permute_b32 v5, v4, v2
+; GFX1164-DPP-NEXT: ds_permute_b32 v6, v4, v3
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[5:6]
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT: v_readlane_b32 s3, v3, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v11, 0
+; GFX1164-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b64 v[9:10], v11, s[0:1]
; GFX1164-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[8:9], v[8:9]
+; GFX1164-DPP-NEXT: v_max_f64 v[7:8], v[9:10], v[9:10]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[6:7], v[0:1]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[7:8], v11, v[7:10], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8
; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6
; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
@@ -6466,56 +6219,51 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, 0x7ff80000 :: v_dual_mov_b32 v2, 0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[4:5], v[2:3]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s0
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v1, s0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_readlane_b32 s2, v2, 31
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_readlane_b32 s3, v3, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s4, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v10, 0
+; GFX1132-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
; GFX1132-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
@@ -6528,9 +6276,9 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB11_2
; GFX1132-DPP-NEXT: .LBB11_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -6567,7 +6315,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_system_scope
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -6575,8 +6324,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_system_scope
; GFX9-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-NEXT: v_max_f32_e32 v0, 4.0, v0
-; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -6676,7 +6425,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_system_scope
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-DPP-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -6684,8 +6434,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_system_scope
; GFX9-DPP-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-DPP-NEXT: v_max_f32_e32 v0, 4.0, v0
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -6789,7 +6539,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_system_scope
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -6797,8 +6548,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_system_scope
; GFX9-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-NEXT: v_max_f32_e32 v0, 4.0, v0
-; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -6898,7 +6649,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_system_scope
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-DPP-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -6906,8 +6658,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_system_scope
; GFX9-DPP-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-DPP-NEXT: v_max_f32_e32 v0, 4.0, v0
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
index 85bcfd88181b9..88324c3a05083 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
@@ -43,7 +43,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_agent_scope_
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -51,8 +52,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_agent_scope_
; GFX9-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-NEXT: v_min_f32_e32 v0, 4.0, v0
-; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -152,7 +153,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_agent_scope_
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-DPP-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -160,8 +162,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_agent_scope_
; GFX9-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-DPP-NEXT: v_min_f32_e32 v0, 4.0, v0
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -269,33 +271,30 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX7LESS-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s2
-; GFX7LESS-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; GFX7LESS-NEXT: v_mul_f32_e64 v2, 1.0, s4
-; GFX7LESS-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB1_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b32 s2, 0x7fc00000
+; GFX7LESS-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s3, s[0:1]
+; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s3
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s4
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX7LESS-NEXT: v_min_f32_e32 v2, s2, v2
+; GFX7LESS-NEXT: s_or_b64 s[4:5], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s2, v2
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB1_4
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: buffer_atomic_fmin v1, off, s[0:3], 0
+; GFX7LESS-NEXT: buffer_atomic_fmin v0, off, s[0:3], 0
; GFX7LESS-NEXT: .LBB1_4:
; GFX7LESS-NEXT: s_endpgm
;
@@ -329,38 +328,36 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x7fc00000
-; GFX9-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_mov_b32 s4, 0x7fc00000
+; GFX9-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s3, v0, s2
-; GFX9-NEXT: v_max_f32_e64 v1, s3, s3
-; GFX9-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX9-NEXT: v_min_f32_e32 v2, v2, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX9-NEXT: v_min_f32_e32 v2, s4, v2
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9-NEXT: s_cbranch_execz .LBB1_5
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX9-NEXT: v_max_f32_e64 v3, s4, s4
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX9-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-NEXT: .LBB1_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-NEXT: v_min_f32_e32 v0, v0, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_min_f32_e32 v0, v0, v3
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -400,30 +397,28 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v1, 0x7fc00000
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX1064-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1064-NEXT: v_max_f32_e64 v2, s3, s3
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX1064-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1064-NEXT: s_mov_b32 s2, 0x7fc00000
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s3, s[0:1]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1064-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1064-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1064-NEXT: v_min_f32_e64 v2, s2, s4
+; GFX1064-NEXT: v_readfirstlane_b32 s2, v2
; GFX1064-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064-NEXT: s_cbranch_execz .LBB1_4
; GFX1064-NEXT: ; %bb.3:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-NEXT: v_mov_b32_e32 v0, s2
+; GFX1064-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_atomic_fmin v0, v1, s[0:1]
+; GFX1064-NEXT: global_atomic_fmin v1, v0, s[0:1]
; GFX1064-NEXT: .LBB1_4:
; GFX1064-NEXT: s_endpgm
;
@@ -457,29 +452,27 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_max_f32_e64 v2, s2, s2
-; GFX1032-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s1, exec_lo
+; GFX1032-NEXT: s_mov_b32 s0, 0x7fc00000
+; GFX1032-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s2, s1
+; GFX1032-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1032-NEXT: s_bitset0_b32 s1, s2
+; GFX1032-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1032-NEXT: v_min_f32_e64 v2, s0, s3
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v2
; GFX1032-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB1_4
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v0, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_atomic_fmin v0, v1, s[0:1]
+; GFX1032-NEXT: global_atomic_fmin v1, v0, s[2:3]
; GFX1032-NEXT: .LBB1_4:
; GFX1032-NEXT: s_endpgm
;
@@ -503,34 +496,31 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0x7fc00000
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
-; GFX1164-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_max_f32_e64 v2, s3, s3
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: s_mov_b32 s2, 0x7fc00000
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1164-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: v_min_f32_e64 v2, s2, s4
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_readfirstlane_b32 s2, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: ; %bb.2:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB1_4
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-NEXT: v_mov_b32_e32 v0, s2
+; GFX1164-NEXT: v_mov_b32_e32 v1, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_atomic_min_f32 v0, v1, s[0:1]
+; GFX1164-NEXT: global_atomic_min_f32 v1, v0, s[0:1]
; GFX1164-NEXT: .LBB1_4:
; GFX1164-NEXT: s_endpgm
;
@@ -555,32 +545,28 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_max_f32_e64 v2, s2, s2
-; GFX1132-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1132-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: s_mov_b32 s0, 0x7fc00000
+; GFX1132-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1132-NEXT: s_bitset0_b32 s1, s2
+; GFX1132-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132-NEXT: v_min_f32_e64 v2, s0, s3
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1132-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB1_4
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v0, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_atomic_min_f32 v0, v1, s[0:1]
+; GFX1132-NEXT: global_atomic_min_f32 v1, v0, s[2:3]
; GFX1132-NEXT: .LBB1_4:
; GFX1132-NEXT: s_endpgm
;
@@ -653,37 +639,19 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[0:1]
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_min_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_min_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_min_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_min_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_min_f32_e32 v4, v4, v5
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v4 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX9-DPP-NEXT: v_min_f32_e32 v3, v4, v3
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
@@ -693,14 +661,14 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_max_f32_e64 v6, s4, s4
+; GFX9-DPP-NEXT: v_max_f32_e64 v4, s4, s4
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-DPP-NEXT: .LBB1_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-DPP-NEXT: v_min_f32_e32 v0, v0, v6
+; GFX9-DPP-NEXT: v_min_f32_e32 v0, v0, v4
; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -742,46 +710,38 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7fc00000, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1064-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1064-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v4, v4
+; GFX1064-DPP-NEXT: ds_permute_b32 v4, v5, v3
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v3, 0
-; GFX1064-DPP-NEXT: v_max_f32_e64 v3, s2, s2
-; GFX1064-DPP-NEXT: v_max_f32_e64 v4, s3, s3
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v4, v3
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b32 s0, s2
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_atomic_fmin v1, v0, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_atomic_fmin v1, v0, s[2:3]
; GFX1064-DPP-NEXT: .LBB1_2:
; GFX1064-DPP-NEXT: s_endpgm
;
@@ -816,38 +776,28 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7fc00000, v0, s0
+; GFX1032-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v3, 31
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s0, s1
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_atomic_fmin v1, v0, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_atomic_fmin v1, v0, s[2:3]
; GFX1032-DPP-NEXT: .LBB1_2:
; GFX1032-DPP-NEXT: s_endpgm
;
@@ -872,56 +822,44 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s[0:1]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v2, v1
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v1, 0x7fc00000, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: v_permlane64_b32 v2, v1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b32 s0, s2
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_atomic_min_f32 v4, v0, s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_min_f32 v4, v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB1_2:
; GFX1164-DPP-NEXT: s_endpgm
;
@@ -947,45 +885,30 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v1, 0x7fc00000, v0, s0
+; GFX1132-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v2, v2 :: v_dual_max_f32 v1, v1, v1
-; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v2, v1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v3, v3 :: v_dual_mov_b32 v3, 0x7fc00000
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v3, v3 :: v_dual_mov_b32 v3, 0x7fc00000
-; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX1132-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v1, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s0, s1
+; GFX1132-DPP-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_atomic_min_f32 v4, v0, s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_min_f32 v3, v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB1_2:
; GFX1132-DPP-NEXT: s_endpgm
%divValue = call float @div.float.value()
@@ -1021,7 +944,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_one_as_scope
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -1029,8 +953,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_one_as_scope
; GFX9-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-NEXT: v_min_f32_e32 v0, 4.0, v0
-; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -1130,7 +1054,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_one_as_scope
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-DPP-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -1138,8 +1063,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_one_as_scope
; GFX9-DPP-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-DPP-NEXT: v_min_f32_e32 v0, 4.0, v0
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -1248,33 +1173,30 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX7LESS-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s2
-; GFX7LESS-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; GFX7LESS-NEXT: v_mul_f32_e64 v2, 1.0, s4
-; GFX7LESS-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB3_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b32 s2, 0x7fc00000
+; GFX7LESS-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s3, s[0:1]
+; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s3
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s4
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX7LESS-NEXT: v_min_f32_e32 v2, s2, v2
+; GFX7LESS-NEXT: s_or_b64 s[4:5], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s2, v2
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB3_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB3_4
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: buffer_atomic_fmin v1, off, s[0:3], 0
+; GFX7LESS-NEXT: buffer_atomic_fmin v0, off, s[0:3], 0
; GFX7LESS-NEXT: .LBB3_4:
; GFX7LESS-NEXT: s_endpgm
;
@@ -1308,38 +1230,36 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x7fc00000
-; GFX9-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_mov_b32 s4, 0x7fc00000
+; GFX9-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s3, v0, s2
-; GFX9-NEXT: v_max_f32_e64 v1, s3, s3
-; GFX9-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX9-NEXT: v_min_f32_e32 v2, v2, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX9-NEXT: v_min_f32_e32 v2, s4, v2
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9-NEXT: s_cbranch_execz .LBB3_5
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX9-NEXT: v_max_f32_e64 v3, s4, s4
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX9-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-NEXT: .LBB3_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-NEXT: v_min_f32_e32 v0, v0, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_min_f32_e32 v0, v0, v3
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -1379,30 +1299,28 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v1, 0x7fc00000
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX1064-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1064-NEXT: v_max_f32_e64 v2, s3, s3
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX1064-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1064-NEXT: s_mov_b32 s2, 0x7fc00000
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s3, s[0:1]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1064-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1064-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1064-NEXT: v_min_f32_e64 v2, s2, s4
+; GFX1064-NEXT: v_readfirstlane_b32 s2, v2
; GFX1064-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064-NEXT: s_cbranch_execz .LBB3_4
; GFX1064-NEXT: ; %bb.3:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-NEXT: v_mov_b32_e32 v0, s2
+; GFX1064-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_atomic_fmin v0, v1, s[0:1]
+; GFX1064-NEXT: global_atomic_fmin v1, v0, s[0:1]
; GFX1064-NEXT: .LBB3_4:
; GFX1064-NEXT: s_endpgm
;
@@ -1436,29 +1354,27 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_max_f32_e64 v2, s2, s2
-; GFX1032-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s1, exec_lo
+; GFX1032-NEXT: s_mov_b32 s0, 0x7fc00000
+; GFX1032-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s2, s1
+; GFX1032-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1032-NEXT: s_bitset0_b32 s1, s2
+; GFX1032-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1032-NEXT: v_min_f32_e64 v2, s0, s3
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v2
; GFX1032-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB3_4
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v0, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_atomic_fmin v0, v1, s[0:1]
+; GFX1032-NEXT: global_atomic_fmin v1, v0, s[2:3]
; GFX1032-NEXT: .LBB3_4:
; GFX1032-NEXT: s_endpgm
;
@@ -1482,34 +1398,31 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0x7fc00000
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
-; GFX1164-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_max_f32_e64 v2, s3, s3
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: s_mov_b32 s2, 0x7fc00000
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1164-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: v_min_f32_e64 v2, s2, s4
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_readfirstlane_b32 s2, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: ; %bb.2:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB3_4
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-NEXT: v_mov_b32_e32 v0, s2
+; GFX1164-NEXT: v_mov_b32_e32 v1, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_atomic_min_f32 v0, v1, s[0:1]
+; GFX1164-NEXT: global_atomic_min_f32 v1, v0, s[0:1]
; GFX1164-NEXT: .LBB3_4:
; GFX1164-NEXT: s_endpgm
;
@@ -1534,32 +1447,28 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_max_f32_e64 v2, s2, s2
-; GFX1132-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1132-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: s_mov_b32 s0, 0x7fc00000
+; GFX1132-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1132-NEXT: s_bitset0_b32 s1, s2
+; GFX1132-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132-NEXT: v_min_f32_e64 v2, s0, s3
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1132-NEXT: s_cbranch_scc1 .LBB3_1
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB3_4
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v0, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_atomic_min_f32 v0, v1, s[0:1]
+; GFX1132-NEXT: global_atomic_min_f32 v1, v0, s[2:3]
; GFX1132-NEXT: .LBB3_4:
; GFX1132-NEXT: s_endpgm
;
@@ -1632,37 +1541,19 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[0:1]
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_min_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_min_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_min_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_min_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_min_f32_e32 v4, v4, v5
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v4 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX9-DPP-NEXT: v_min_f32_e32 v3, v4, v3
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
@@ -1672,14 +1563,14 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_max_f32_e64 v6, s4, s4
+; GFX9-DPP-NEXT: v_max_f32_e64 v4, s4, s4
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-DPP-NEXT: .LBB3_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-DPP-NEXT: v_min_f32_e32 v0, v0, v6
+; GFX9-DPP-NEXT: v_min_f32_e32 v0, v0, v4
; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -1721,46 +1612,38 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7fc00000, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1064-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1064-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v4, v4
+; GFX1064-DPP-NEXT: ds_permute_b32 v4, v5, v3
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v3, 0
-; GFX1064-DPP-NEXT: v_max_f32_e64 v3, s2, s2
-; GFX1064-DPP-NEXT: v_max_f32_e64 v4, s3, s3
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v4, v3
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b32 s0, s2
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_atomic_fmin v1, v0, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_atomic_fmin v1, v0, s[2:3]
; GFX1064-DPP-NEXT: .LBB3_2:
; GFX1064-DPP-NEXT: s_endpgm
;
@@ -1795,38 +1678,28 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7fc00000, v0, s0
+; GFX1032-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v3, 31
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s0, s1
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_atomic_fmin v1, v0, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_atomic_fmin v1, v0, s[2:3]
; GFX1032-DPP-NEXT: .LBB3_2:
; GFX1032-DPP-NEXT: s_endpgm
;
@@ -1851,56 +1724,44 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s[0:1]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v2, v1
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v1, 0x7fc00000, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: v_permlane64_b32 v2, v1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b32 s0, s2
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_atomic_min_f32 v4, v0, s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_min_f32 v4, v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB3_2:
; GFX1164-DPP-NEXT: s_endpgm
;
@@ -1926,45 +1787,30 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v2, v2 :: v_dual_max_f32 v1, v1, v1
-; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v2, v1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v3, v3 :: v_dual_mov_b32 v3, 0x7fc00000
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v1, 0x7fc00000, v0, s0
+; GFX1132-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v3, v3 :: v_dual_mov_b32 v3, 0x7fc00000
-; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v1, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s0, s1
+; GFX1132-DPP-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_atomic_min_f32 v4, v0, s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_min_f32 v3, v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB3_2:
; GFX1132-DPP-NEXT: s_endpgm
%divValue = call float @div.float.value()
@@ -2001,7 +1847,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_default_scop
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -2009,8 +1856,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_default_scop
; GFX9-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-NEXT: v_min_f32_e32 v0, 4.0, v0
-; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -2110,7 +1957,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_default_scop
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-DPP-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -2118,8 +1966,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_default_scop
; GFX9-DPP-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-DPP-NEXT: v_min_f32_e32 v0, 4.0, v0
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -2227,33 +2075,30 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX7LESS-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s2
-; GFX7LESS-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; GFX7LESS-NEXT: v_mul_f32_e64 v2, 1.0, s4
-; GFX7LESS-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB5_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b32 s2, 0x7fc00000
+; GFX7LESS-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s3, s[0:1]
+; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s3
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s4
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX7LESS-NEXT: v_min_f32_e32 v2, s2, v2
+; GFX7LESS-NEXT: s_or_b64 s[4:5], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s2, v2
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB5_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB5_4
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: buffer_atomic_fmin v1, off, s[0:3], 0
+; GFX7LESS-NEXT: buffer_atomic_fmin v0, off, s[0:3], 0
; GFX7LESS-NEXT: .LBB5_4:
; GFX7LESS-NEXT: s_endpgm
;
@@ -2287,38 +2132,36 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x7fc00000
-; GFX9-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_mov_b32 s4, 0x7fc00000
+; GFX9-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s3, v0, s2
-; GFX9-NEXT: v_max_f32_e64 v1, s3, s3
-; GFX9-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX9-NEXT: v_min_f32_e32 v2, v2, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX9-NEXT: v_min_f32_e32 v2, s4, v2
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9-NEXT: s_cbranch_execz .LBB5_5
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX9-NEXT: v_max_f32_e64 v3, s4, s4
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX9-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-NEXT: .LBB5_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-NEXT: v_min_f32_e32 v0, v0, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_min_f32_e32 v0, v0, v3
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -2358,30 +2201,28 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v1, 0x7fc00000
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX1064-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1064-NEXT: v_max_f32_e64 v2, s3, s3
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX1064-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1064-NEXT: s_mov_b32 s2, 0x7fc00000
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s3, s[0:1]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1064-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1064-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1064-NEXT: v_min_f32_e64 v2, s2, s4
+; GFX1064-NEXT: v_readfirstlane_b32 s2, v2
; GFX1064-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064-NEXT: s_cbranch_execz .LBB5_4
; GFX1064-NEXT: ; %bb.3:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-NEXT: v_mov_b32_e32 v0, s2
+; GFX1064-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_atomic_fmin v0, v1, s[0:1]
+; GFX1064-NEXT: global_atomic_fmin v1, v0, s[0:1]
; GFX1064-NEXT: .LBB5_4:
; GFX1064-NEXT: s_endpgm
;
@@ -2415,29 +2256,27 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_max_f32_e64 v2, s2, s2
-; GFX1032-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s1, exec_lo
+; GFX1032-NEXT: s_mov_b32 s0, 0x7fc00000
+; GFX1032-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s2, s1
+; GFX1032-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1032-NEXT: s_bitset0_b32 s1, s2
+; GFX1032-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1032-NEXT: v_min_f32_e64 v2, s0, s3
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v2
; GFX1032-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB5_4
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v0, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_atomic_fmin v0, v1, s[0:1]
+; GFX1032-NEXT: global_atomic_fmin v1, v0, s[2:3]
; GFX1032-NEXT: .LBB5_4:
; GFX1032-NEXT: s_endpgm
;
@@ -2461,34 +2300,31 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0x7fc00000
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
-; GFX1164-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_max_f32_e64 v2, s3, s3
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: s_mov_b32 s2, 0x7fc00000
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1164-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: v_min_f32_e64 v2, s2, s4
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_readfirstlane_b32 s2, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: ; %bb.2:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB5_4
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-NEXT: v_mov_b32_e32 v0, s2
+; GFX1164-NEXT: v_mov_b32_e32 v1, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_atomic_min_f32 v0, v1, s[0:1]
+; GFX1164-NEXT: global_atomic_min_f32 v1, v0, s[0:1]
; GFX1164-NEXT: .LBB5_4:
; GFX1164-NEXT: s_endpgm
;
@@ -2513,32 +2349,28 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_max_f32_e64 v2, s2, s2
-; GFX1132-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1132-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: s_mov_b32 s0, 0x7fc00000
+; GFX1132-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1132-NEXT: s_bitset0_b32 s1, s2
+; GFX1132-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132-NEXT: v_min_f32_e64 v2, s0, s3
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1132-NEXT: s_cbranch_scc1 .LBB5_1
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB5_4
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v0, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_atomic_min_f32 v0, v1, s[0:1]
+; GFX1132-NEXT: global_atomic_min_f32 v1, v0, s[2:3]
; GFX1132-NEXT: .LBB5_4:
; GFX1132-NEXT: s_endpgm
;
@@ -2611,37 +2443,19 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[0:1]
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_min_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_min_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_min_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_min_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_min_f32_e32 v4, v4, v5
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v4 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX9-DPP-NEXT: v_min_f32_e32 v3, v4, v3
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
@@ -2651,14 +2465,14 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_max_f32_e64 v6, s4, s4
+; GFX9-DPP-NEXT: v_max_f32_e64 v4, s4, s4
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-DPP-NEXT: .LBB5_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-DPP-NEXT: v_min_f32_e32 v0, v0, v6
+; GFX9-DPP-NEXT: v_min_f32_e32 v0, v0, v4
; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -2700,46 +2514,38 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7fc00000, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1064-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1064-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v4, v4
+; GFX1064-DPP-NEXT: ds_permute_b32 v4, v5, v3
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v3, 0
-; GFX1064-DPP-NEXT: v_max_f32_e64 v3, s2, s2
-; GFX1064-DPP-NEXT: v_max_f32_e64 v4, s3, s3
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v4, v3
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b32 s0, s2
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_atomic_fmin v1, v0, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_atomic_fmin v1, v0, s[2:3]
; GFX1064-DPP-NEXT: .LBB5_2:
; GFX1064-DPP-NEXT: s_endpgm
;
@@ -2774,38 +2580,28 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7fc00000, v0, s0
+; GFX1032-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v3, 31
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s0, s1
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_atomic_fmin v1, v0, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_atomic_fmin v1, v0, s[2:3]
; GFX1032-DPP-NEXT: .LBB5_2:
; GFX1032-DPP-NEXT: s_endpgm
;
@@ -2830,56 +2626,44 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s[0:1]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v2, v1
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v1, 0x7fc00000, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: v_permlane64_b32 v2, v1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b32 s0, s2
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_atomic_min_f32 v4, v0, s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_min_f32 v4, v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB5_2:
; GFX1164-DPP-NEXT: s_endpgm
;
@@ -2905,45 +2689,30 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v1, 0x7fc00000, v0, s0
+; GFX1132-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v2, v2 :: v_dual_max_f32 v1, v1, v1
-; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v2, v1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v3, v3 :: v_dual_mov_b32 v3, 0x7fc00000
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v3, v3 :: v_dual_mov_b32 v3, 0x7fc00000
-; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX1132-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v1, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s0, s1
+; GFX1132-DPP-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_atomic_min_f32 v4, v0, s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_min_f32 v3, v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB5_2:
; GFX1132-DPP-NEXT: s_endpgm
%divValue = call float @div.float.value()
@@ -2979,8 +2748,9 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
; GFX9-NEXT: s_cbranch_execz .LBB6_3
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_max_f64 v[4:5], 4.0, 4.0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: v_mov_b32_e32 v6, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -2989,8 +2759,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
; GFX9-NEXT: .LBB6_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
@@ -3044,7 +2814,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
; GFX1164-NEXT: s_cbranch_execz .LBB6_3
; GFX1164-NEXT: ; %bb.1:
; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1164-NEXT: v_mov_b32_e32 v6, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
@@ -3055,8 +2826,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1164-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
@@ -3078,7 +2849,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
; GFX1132-NEXT: s_cbranch_execz .LBB6_3
; GFX1132-NEXT: ; %bb.1:
; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1132-NEXT: v_mov_b32_e32 v6, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
@@ -3087,8 +2859,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1132-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
@@ -3126,8 +2898,9 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
; GFX9-DPP-NEXT: s_cbranch_execz .LBB6_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -3136,8 +2909,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
; GFX9-DPP-NEXT: .LBB6_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX9-DPP-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX9-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1
@@ -3191,7 +2964,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB6_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -3202,8 +2976,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1164-DPP-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1
@@ -3225,7 +2999,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB6_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -3234,8 +3009,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1132-DPP-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
@@ -3281,35 +3056,35 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX7LESS-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s4
-; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s4
-; GFX7LESS-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX7LESS-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX7LESS-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB7_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: s_mov_b32 s0, 0
+; GFX7LESS-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
+; GFX7LESS-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s6
+; GFX7LESS-NEXT: v_readlane_b32 s5, v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, s1
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s0
+; GFX7LESS-NEXT: v_min_f64 v[3:4], s[4:5], v[3:4]
+; GFX7LESS-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX7LESS-NEXT: v_readfirstlane_b32 s0, v3
+; GFX7LESS-NEXT: s_or_b64 s[4:5], s[2:3], s[2:3]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s1, v4
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB7_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB7_4
; GFX7LESS-NEXT: ; %bb.3:
-; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
-; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[34:35], 0x9
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s0
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s1
+; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT: s_mov_b32 s6, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: buffer_atomic_fmin_x2 v[2:3], off, s[0:3], 0
+; GFX7LESS-NEXT: buffer_atomic_fmin_x2 v[0:1], off, s[4:7], 0
; GFX7LESS-NEXT: .LBB7_4:
; GFX7LESS-NEXT: s_endpgm
;
@@ -3343,46 +3118,46 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX9-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s3, v1, s4
-; GFX9-NEXT: v_readlane_b32 s2, v0, s4
-; GFX9-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX9-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX9-NEXT: v_min_f64 v[4:5], v[2:3], v[4:5]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: s_mov_b32 s0, 0
+; GFX9-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX9-NEXT: s_mov_b64 s[2:3], exec
+; GFX9-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9-NEXT: v_readlane_b32 s4, v0, s6
+; GFX9-NEXT: v_readlane_b32 s5, v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_min_f64 v[3:4], s[4:5], v[3:4]
+; GFX9-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v3
+; GFX9-NEXT: v_readfirstlane_b32 s1, v4
; GFX9-NEXT: s_cbranch_scc1 .LBB7_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB7_5
; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX9-NEXT: s_mov_b64 s[0:1], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[2:3]
; GFX9-NEXT: .LBB7_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_cbranch_execnz .LBB7_4
; GFX9-NEXT: .LBB7_5:
; GFX9-NEXT: s_endpgm
@@ -3417,32 +3192,32 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX1064-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1064-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1064-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX1064-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_mov_b32 s0, 0
+; GFX1064-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1064-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1064-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1064-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064-NEXT: v_min_f64 v[3:4], s[4:5], s[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v4
; GFX1064-NEXT: s_cbranch_scc1 .LBB7_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB7_4
; GFX1064-NEXT: ; %bb.3:
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v0, s0
+; GFX1064-NEXT: v_mov_b32_e32 v1, s1
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_atomic_fmin_x2 v0, v[2:3], s[0:1]
+; GFX1064-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[2:3]
; GFX1064-NEXT: .LBB7_4:
; GFX1064-NEXT: s_endpgm
;
@@ -3476,31 +3251,31 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1032-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1032-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, 0
+; GFX1032-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1032-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1032-NEXT: s_bitset0_b32 s2, s3
+; GFX1032-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032-NEXT: v_min_f64 v[3:4], s[4:5], s[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v4
; GFX1032-NEXT: s_cbranch_scc1 .LBB7_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB7_4
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v0, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_atomic_fmin_x2 v0, v[2:3], s[0:1]
+; GFX1032-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[2:3]
; GFX1032-NEXT: .LBB7_4:
; GFX1032-NEXT: s_endpgm
;
@@ -3524,52 +3299,49 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX1164-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1164-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_mov_b32 s0, 0
+; GFX1164-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_min_f64 v[4:5], v[2:3], v[4:5]
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s6, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1164-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1164-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164-NEXT: v_min_f64 v[3:4], s[4:5], s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v4
; GFX1164-NEXT: s_cbranch_scc1 .LBB7_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: ; %bb.2:
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB7_5
; GFX1164-NEXT: ; %bb.3:
-; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[0:1], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[2:3]
; GFX1164-NEXT: .LBB7_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1]
; GFX1164-NEXT: s_cbranch_execnz .LBB7_4
; GFX1164-NEXT: .LBB7_5:
; GFX1164-NEXT: s_endpgm
@@ -3595,49 +3367,47 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 0x7ff80000
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX1132-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s0, 0
+; GFX1132-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1132-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1132-NEXT: s_bitset0_b32 s2, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1132-NEXT: v_min_f64 v[4:5], v[2:3], v[4:5]
+; GFX1132-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132-NEXT: v_min_f64 v[3:4], s[4:5], s[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v4
; GFX1132-NEXT: s_cbranch_scc1 .LBB7_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB7_5
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
; GFX1132-NEXT: v_mov_b32_e32 v6, 0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX1132-NEXT: s_mov_b32 s0, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[2:3]
; GFX1132-NEXT: .LBB7_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX1132-NEXT: s_cbranch_execnz .LBB7_4
; GFX1132-NEXT: .LBB7_5:
; GFX1132-NEXT: s_endpgm
@@ -3708,80 +3478,74 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1]
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0x7ff80000
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, v3, v1, s[0:1]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v7 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[4:5], v[6:7]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
-; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
+; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3]
+; GFX9-DPP-NEXT: global_load_dwordx2 v[10:11], v2, s[0:1]
; GFX9-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_max_f64 v[1:2], s[0:1], s[0:1]
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_max_f64 v[9:10], v[11:12], v[11:12]
-; GFX9-DPP-NEXT: v_min_f64 v[9:10], v[9:10], v[1:2]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[10:11], v[10:11]
+; GFX9-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[8:9], v2, v[8:11], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v2
-; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v1
-; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[10:11]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v9
+; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v10, v8
+; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB7_2
; GFX9-DPP-NEXT: .LBB7_3:
; GFX9-DPP-NEXT: s_endpgm
@@ -3817,57 +3581,59 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x7ff80000, v1, s[0:1]
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7ff80000, v1, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s5, v4, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], s[4:5], s[4:5]
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], s[2:3], s[2:3]
-; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[5:6], v[3:4]
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v7
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: ds_permute_b32 v6, v5, v3
+; GFX1064-DPP-NEXT: ds_permute_b32 v7, v5, v4
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[6:7]
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 63
+; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB7_2
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s1
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[2:3]
; GFX1064-DPP-NEXT: .LBB7_2:
; GFX1064-DPP-NEXT: s_endpgm
;
@@ -3901,50 +3667,48 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x7ff80000, v1, s0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[5:6], v[3:4]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s2
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7ff80000, v1, s2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT: v_readlane_b32 s0, v3, 31
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v4, 31
+; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB7_2
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[2:3]
; GFX1032-DPP-NEXT: .LBB7_2:
; GFX1032-DPP-NEXT: s_endpgm
;
@@ -3969,82 +3733,85 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s[0:1]
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v1, s[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[4:5], v[2:3]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v5, v3
-; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, -1, v4
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v4, 32, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v4, 4, v4
+; GFX1164-DPP-NEXT: ds_permute_b32 v5, v4, v2
+; GFX1164-DPP-NEXT: ds_permute_b32 v6, v4, v3
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[5:6]
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT: v_readlane_b32 s3, v3, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v11, 0
+; GFX1164-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b64 v[9:10], v11, s[0:1]
; GFX1164-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[8:9], v[8:9]
+; GFX1164-DPP-NEXT: v_max_f64 v[7:8], v[9:10], v[9:10]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_min_f64 v[6:7], v[6:7], v[0:1]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT: v_min_f64 v[7:8], v[7:8], v[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[7:8], v11, v[7:10], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8
; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6
; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
@@ -4074,56 +3841,51 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, 0x7ff80000 :: v_dual_mov_b32 v2, 0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[4:5], v[2:3]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s0
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v1, s0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_readlane_b32 s2, v2, 31
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_readlane_b32 s3, v3, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s4, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v10, 0
+; GFX1132-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
; GFX1132-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
@@ -4136,9 +3898,9 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB7_2
; GFX1132-DPP-NEXT: .LBB7_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -4175,8 +3937,9 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_a
; GFX9-NEXT: s_cbranch_execz .LBB8_3
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_max_f64 v[4:5], 4.0, 4.0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: v_mov_b32_e32 v6, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -4185,8 +3948,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_a
; GFX9-NEXT: .LBB8_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
@@ -4240,7 +4003,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_a
; GFX1164-NEXT: s_cbranch_execz .LBB8_3
; GFX1164-NEXT: ; %bb.1:
; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1164-NEXT: v_mov_b32_e32 v6, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
@@ -4251,8 +4015,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_a
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1164-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
@@ -4274,7 +4038,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_a
; GFX1132-NEXT: s_cbranch_execz .LBB8_3
; GFX1132-NEXT: ; %bb.1:
; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1132-NEXT: v_mov_b32_e32 v6, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
@@ -4283,8 +4048,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_a
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1132-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
@@ -4322,8 +4087,9 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_a
; GFX9-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -4332,8 +4098,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_a
; GFX9-DPP-NEXT: .LBB8_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX9-DPP-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX9-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1
@@ -4387,7 +4153,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_a
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -4398,8 +4165,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_a
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1164-DPP-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1
@@ -4421,7 +4188,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_a
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -4430,8 +4198,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_a
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1132-DPP-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
@@ -4477,35 +4245,35 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX7LESS-NEXT: .LBB9_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s4
-; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s4
-; GFX7LESS-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX7LESS-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX7LESS-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB9_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: s_mov_b32 s0, 0
+; GFX7LESS-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
+; GFX7LESS-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s6
+; GFX7LESS-NEXT: v_readlane_b32 s5, v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, s1
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s0
+; GFX7LESS-NEXT: v_min_f64 v[3:4], s[4:5], v[3:4]
+; GFX7LESS-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX7LESS-NEXT: v_readfirstlane_b32 s0, v3
+; GFX7LESS-NEXT: s_or_b64 s[4:5], s[2:3], s[2:3]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s1, v4
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB9_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB9_4
; GFX7LESS-NEXT: ; %bb.3:
-; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
-; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[34:35], 0x9
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s0
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s1
+; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT: s_mov_b32 s6, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: buffer_atomic_fmin_x2 v[2:3], off, s[0:3], 0
+; GFX7LESS-NEXT: buffer_atomic_fmin_x2 v[0:1], off, s[4:7], 0
; GFX7LESS-NEXT: .LBB9_4:
; GFX7LESS-NEXT: s_endpgm
;
@@ -4539,46 +4307,46 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX9-NEXT: .LBB9_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s3, v1, s4
-; GFX9-NEXT: v_readlane_b32 s2, v0, s4
-; GFX9-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX9-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX9-NEXT: v_min_f64 v[4:5], v[2:3], v[4:5]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: s_mov_b32 s0, 0
+; GFX9-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX9-NEXT: s_mov_b64 s[2:3], exec
+; GFX9-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9-NEXT: v_readlane_b32 s4, v0, s6
+; GFX9-NEXT: v_readlane_b32 s5, v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_min_f64 v[3:4], s[4:5], v[3:4]
+; GFX9-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v3
+; GFX9-NEXT: v_readfirstlane_b32 s1, v4
; GFX9-NEXT: s_cbranch_scc1 .LBB9_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB9_5
; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX9-NEXT: s_mov_b64 s[0:1], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[2:3]
; GFX9-NEXT: .LBB9_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_cbranch_execnz .LBB9_4
; GFX9-NEXT: .LBB9_5:
; GFX9-NEXT: s_endpgm
@@ -4613,32 +4381,32 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB9_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX1064-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1064-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1064-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX1064-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_mov_b32 s0, 0
+; GFX1064-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1064-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1064-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1064-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064-NEXT: v_min_f64 v[3:4], s[4:5], s[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v4
; GFX1064-NEXT: s_cbranch_scc1 .LBB9_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB9_4
; GFX1064-NEXT: ; %bb.3:
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v0, s0
+; GFX1064-NEXT: v_mov_b32_e32 v1, s1
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_atomic_fmin_x2 v0, v[2:3], s[0:1]
+; GFX1064-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[2:3]
; GFX1064-NEXT: .LBB9_4:
; GFX1064-NEXT: s_endpgm
;
@@ -4672,31 +4440,31 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB9_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1032-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1032-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, 0
+; GFX1032-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1032-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1032-NEXT: s_bitset0_b32 s2, s3
+; GFX1032-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032-NEXT: v_min_f64 v[3:4], s[4:5], s[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v4
; GFX1032-NEXT: s_cbranch_scc1 .LBB9_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB9_4
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v0, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_atomic_fmin_x2 v0, v[2:3], s[0:1]
+; GFX1032-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[2:3]
; GFX1032-NEXT: .LBB9_4:
; GFX1032-NEXT: s_endpgm
;
@@ -4720,52 +4488,49 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB9_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX1164-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1164-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_mov_b32 s0, 0
+; GFX1164-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_min_f64 v[4:5], v[2:3], v[4:5]
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s6, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1164-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1164-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164-NEXT: v_min_f64 v[3:4], s[4:5], s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v4
; GFX1164-NEXT: s_cbranch_scc1 .LBB9_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: ; %bb.2:
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB9_5
; GFX1164-NEXT: ; %bb.3:
-; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[0:1], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[2:3]
; GFX1164-NEXT: .LBB9_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1]
; GFX1164-NEXT: s_cbranch_execnz .LBB9_4
; GFX1164-NEXT: .LBB9_5:
; GFX1164-NEXT: s_endpgm
@@ -4791,49 +4556,47 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 0x7ff80000
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB9_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX1132-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s0, 0
+; GFX1132-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1132-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1132-NEXT: s_bitset0_b32 s2, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1132-NEXT: v_min_f64 v[4:5], v[2:3], v[4:5]
+; GFX1132-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132-NEXT: v_min_f64 v[3:4], s[4:5], s[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v4
; GFX1132-NEXT: s_cbranch_scc1 .LBB9_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB9_5
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
; GFX1132-NEXT: v_mov_b32_e32 v6, 0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX1132-NEXT: s_mov_b32 s0, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[2:3]
; GFX1132-NEXT: .LBB9_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX1132-NEXT: s_cbranch_execnz .LBB9_4
; GFX1132-NEXT: .LBB9_5:
; GFX1132-NEXT: s_endpgm
@@ -4904,80 +4667,74 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1]
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0x7ff80000
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, v3, v1, s[0:1]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v7 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[4:5], v[6:7]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
-; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
+; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3]
+; GFX9-DPP-NEXT: global_load_dwordx2 v[10:11], v2, s[0:1]
; GFX9-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_max_f64 v[1:2], s[0:1], s[0:1]
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_max_f64 v[9:10], v[11:12], v[11:12]
-; GFX9-DPP-NEXT: v_min_f64 v[9:10], v[9:10], v[1:2]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[10:11], v[10:11]
+; GFX9-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[8:9], v2, v[8:11], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v2
-; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v1
-; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[10:11]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v9
+; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v10, v8
+; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB9_2
; GFX9-DPP-NEXT: .LBB9_3:
; GFX9-DPP-NEXT: s_endpgm
@@ -5013,57 +4770,59 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x7ff80000, v1, s[0:1]
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7ff80000, v1, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s5, v4, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], s[4:5], s[4:5]
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], s[2:3], s[2:3]
-; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[5:6], v[3:4]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v7
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: ds_permute_b32 v6, v5, v3
+; GFX1064-DPP-NEXT: ds_permute_b32 v7, v5, v4
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[6:7]
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 63
+; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB9_2
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s1
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[2:3]
; GFX1064-DPP-NEXT: .LBB9_2:
; GFX1064-DPP-NEXT: s_endpgm
;
@@ -5097,50 +4856,48 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x7ff80000, v1, s0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[5:6], v[3:4]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s2
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7ff80000, v1, s2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT: v_readlane_b32 s0, v3, 31
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v4, 31
+; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB9_2
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[2:3]
; GFX1032-DPP-NEXT: .LBB9_2:
; GFX1032-DPP-NEXT: s_endpgm
;
@@ -5165,82 +4922,85 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s[0:1]
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v1, s[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[4:5], v[2:3]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v5, v3
-; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, -1, v4
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v4, 32, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v4, 4, v4
+; GFX1164-DPP-NEXT: ds_permute_b32 v5, v4, v2
+; GFX1164-DPP-NEXT: ds_permute_b32 v6, v4, v3
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[5:6]
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT: v_readlane_b32 s3, v3, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v11, 0
+; GFX1164-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b64 v[9:10], v11, s[0:1]
; GFX1164-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[8:9], v[8:9]
+; GFX1164-DPP-NEXT: v_max_f64 v[7:8], v[9:10], v[9:10]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_min_f64 v[6:7], v[6:7], v[0:1]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT: v_min_f64 v[7:8], v[7:8], v[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[7:8], v11, v[7:10], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8
; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6
; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
@@ -5270,56 +5030,51 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, 0x7ff80000 :: v_dual_mov_b32 v2, 0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[4:5], v[2:3]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s0
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v1, s0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_readlane_b32 s2, v2, 31
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_readlane_b32 s3, v3, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s4, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v10, 0
+; GFX1132-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
; GFX1132-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
@@ -5332,9 +5087,9 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB9_2
; GFX1132-DPP-NEXT: .LBB9_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -5371,8 +5126,9 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
; GFX9-NEXT: s_cbranch_execz .LBB10_3
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_max_f64 v[4:5], 4.0, 4.0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: v_mov_b32_e32 v6, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -5381,8 +5137,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
; GFX9-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
@@ -5436,7 +5192,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
; GFX1164-NEXT: s_cbranch_execz .LBB10_3
; GFX1164-NEXT: ; %bb.1:
; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1164-NEXT: v_mov_b32_e32 v6, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
@@ -5447,8 +5204,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1164-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
@@ -5470,7 +5227,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
; GFX1132-NEXT: s_cbranch_execz .LBB10_3
; GFX1132-NEXT: ; %bb.1:
; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1132-NEXT: v_mov_b32_e32 v6, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
@@ -5479,8 +5237,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1132-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
@@ -5518,8 +5276,9 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
; GFX9-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -5528,8 +5287,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
; GFX9-DPP-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX9-DPP-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX9-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1
@@ -5583,7 +5342,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -5594,8 +5354,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1164-DPP-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1
@@ -5617,7 +5377,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -5626,8 +5387,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1132-DPP-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
@@ -5673,35 +5434,35 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX7LESS-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s4
-; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s4
-; GFX7LESS-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX7LESS-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX7LESS-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB11_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: s_mov_b32 s0, 0
+; GFX7LESS-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
+; GFX7LESS-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s6
+; GFX7LESS-NEXT: v_readlane_b32 s5, v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, s1
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s0
+; GFX7LESS-NEXT: v_min_f64 v[3:4], s[4:5], v[3:4]
+; GFX7LESS-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX7LESS-NEXT: v_readfirstlane_b32 s0, v3
+; GFX7LESS-NEXT: s_or_b64 s[4:5], s[2:3], s[2:3]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s1, v4
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB11_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB11_4
; GFX7LESS-NEXT: ; %bb.3:
-; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
-; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[34:35], 0x9
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s0
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s1
+; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT: s_mov_b32 s6, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: buffer_atomic_fmin_x2 v[2:3], off, s[0:3], 0
+; GFX7LESS-NEXT: buffer_atomic_fmin_x2 v[0:1], off, s[4:7], 0
; GFX7LESS-NEXT: .LBB11_4:
; GFX7LESS-NEXT: s_endpgm
;
@@ -5735,46 +5496,46 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX9-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s3, v1, s4
-; GFX9-NEXT: v_readlane_b32 s2, v0, s4
-; GFX9-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX9-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX9-NEXT: v_min_f64 v[4:5], v[2:3], v[4:5]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: s_mov_b32 s0, 0
+; GFX9-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX9-NEXT: s_mov_b64 s[2:3], exec
+; GFX9-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9-NEXT: v_readlane_b32 s4, v0, s6
+; GFX9-NEXT: v_readlane_b32 s5, v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_min_f64 v[3:4], s[4:5], v[3:4]
+; GFX9-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v3
+; GFX9-NEXT: v_readfirstlane_b32 s1, v4
; GFX9-NEXT: s_cbranch_scc1 .LBB11_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB11_5
; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX9-NEXT: s_mov_b64 s[0:1], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[2:3]
; GFX9-NEXT: .LBB11_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_cbranch_execnz .LBB11_4
; GFX9-NEXT: .LBB11_5:
; GFX9-NEXT: s_endpgm
@@ -5809,32 +5570,32 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX1064-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1064-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1064-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX1064-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_mov_b32 s0, 0
+; GFX1064-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1064-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1064-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1064-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064-NEXT: v_min_f64 v[3:4], s[4:5], s[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v4
; GFX1064-NEXT: s_cbranch_scc1 .LBB11_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB11_4
; GFX1064-NEXT: ; %bb.3:
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v0, s0
+; GFX1064-NEXT: v_mov_b32_e32 v1, s1
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_atomic_fmin_x2 v0, v[2:3], s[0:1]
+; GFX1064-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[2:3]
; GFX1064-NEXT: .LBB11_4:
; GFX1064-NEXT: s_endpgm
;
@@ -5868,31 +5629,31 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1032-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1032-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, 0
+; GFX1032-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1032-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1032-NEXT: s_bitset0_b32 s2, s3
+; GFX1032-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032-NEXT: v_min_f64 v[3:4], s[4:5], s[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v4
; GFX1032-NEXT: s_cbranch_scc1 .LBB11_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB11_4
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v0, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_atomic_fmin_x2 v0, v[2:3], s[0:1]
+; GFX1032-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[2:3]
; GFX1032-NEXT: .LBB11_4:
; GFX1032-NEXT: s_endpgm
;
@@ -5916,52 +5677,49 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX1164-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1164-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_mov_b32 s0, 0
+; GFX1164-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_min_f64 v[4:5], v[2:3], v[4:5]
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s6, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1164-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1164-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164-NEXT: v_min_f64 v[3:4], s[4:5], s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v4
; GFX1164-NEXT: s_cbranch_scc1 .LBB11_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: ; %bb.2:
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB11_5
; GFX1164-NEXT: ; %bb.3:
-; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[0:1], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[2:3]
; GFX1164-NEXT: .LBB11_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1]
; GFX1164-NEXT: s_cbranch_execnz .LBB11_4
; GFX1164-NEXT: .LBB11_5:
; GFX1164-NEXT: s_endpgm
@@ -5987,49 +5745,47 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 0x7ff80000
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX1132-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s0, 0
+; GFX1132-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1132-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1132-NEXT: s_bitset0_b32 s2, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1132-NEXT: v_min_f64 v[4:5], v[2:3], v[4:5]
+; GFX1132-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132-NEXT: v_min_f64 v[3:4], s[4:5], s[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v4
; GFX1132-NEXT: s_cbranch_scc1 .LBB11_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB11_5
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
; GFX1132-NEXT: v_mov_b32_e32 v6, 0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX1132-NEXT: s_mov_b32 s0, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[2:3]
; GFX1132-NEXT: .LBB11_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX1132-NEXT: s_cbranch_execnz .LBB11_4
; GFX1132-NEXT: .LBB11_5:
; GFX1132-NEXT: s_endpgm
@@ -6100,80 +5856,74 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1]
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0x7ff80000
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, v3, v1, s[0:1]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v7 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[4:5], v[6:7]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
-; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
+; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3]
+; GFX9-DPP-NEXT: global_load_dwordx2 v[10:11], v2, s[0:1]
; GFX9-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_max_f64 v[1:2], s[0:1], s[0:1]
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_max_f64 v[9:10], v[11:12], v[11:12]
-; GFX9-DPP-NEXT: v_min_f64 v[9:10], v[9:10], v[1:2]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[10:11], v[10:11]
+; GFX9-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[8:9], v2, v[8:11], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v2
-; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v1
-; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[10:11]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v9
+; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v10, v8
+; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB11_2
; GFX9-DPP-NEXT: .LBB11_3:
; GFX9-DPP-NEXT: s_endpgm
@@ -6209,57 +5959,59 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x7ff80000, v1, s[0:1]
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7ff80000, v1, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s5, v4, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], s[4:5], s[4:5]
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], s[2:3], s[2:3]
-; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[5:6], v[3:4]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v7
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: ds_permute_b32 v6, v5, v3
+; GFX1064-DPP-NEXT: ds_permute_b32 v7, v5, v4
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[6:7]
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 63
+; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB11_2
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s1
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[2:3]
; GFX1064-DPP-NEXT: .LBB11_2:
; GFX1064-DPP-NEXT: s_endpgm
;
@@ -6293,50 +6045,48 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x7ff80000, v1, s0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[5:6], v[3:4]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s2
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7ff80000, v1, s2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT: v_readlane_b32 s0, v3, 31
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v4, 31
+; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB11_2
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[2:3]
; GFX1032-DPP-NEXT: .LBB11_2:
; GFX1032-DPP-NEXT: s_endpgm
;
@@ -6361,82 +6111,85 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s[0:1]
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v1, s[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[4:5], v[2:3]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v5, v3
-; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, -1, v4
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v4, 32, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v4, 4, v4
+; GFX1164-DPP-NEXT: ds_permute_b32 v5, v4, v2
+; GFX1164-DPP-NEXT: ds_permute_b32 v6, v4, v3
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[5:6]
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT: v_readlane_b32 s3, v3, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v11, 0
+; GFX1164-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b64 v[9:10], v11, s[0:1]
; GFX1164-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[8:9], v[8:9]
+; GFX1164-DPP-NEXT: v_max_f64 v[7:8], v[9:10], v[9:10]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_min_f64 v[6:7], v[6:7], v[0:1]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT: v_min_f64 v[7:8], v[7:8], v[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[7:8], v11, v[7:10], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8
; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6
; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
@@ -6466,56 +6219,51 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, 0x7ff80000 :: v_dual_mov_b32 v2, 0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[4:5], v[2:3]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s0
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v1, s0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_readlane_b32 s2, v2, 31
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_readlane_b32 s3, v3, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s4, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v10, 0
+; GFX1132-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
; GFX1132-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
@@ -6528,9 +6276,9 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB11_2
; GFX1132-DPP-NEXT: .LBB11_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -6567,7 +6315,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_system_scope
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -6575,8 +6324,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_system_scope
; GFX9-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-NEXT: v_min_f32_e32 v0, 4.0, v0
-; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -6676,7 +6425,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_system_scope
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-DPP-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -6684,8 +6434,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_system_scope
; GFX9-DPP-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-DPP-NEXT: v_min_f32_e32 v0, 4.0, v0
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -6789,7 +6539,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_system_scope
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -6797,8 +6548,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_system_scope
; GFX9-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-NEXT: v_min_f32_e32 v0, 4.0, v0
-; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -6898,7 +6649,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_system_scope
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-DPP-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -6906,8 +6658,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_system_scope
; GFX9-DPP-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-DPP-NEXT: v_min_f32_e32 v0, 4.0, v0
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
index 327928e152f95..f3a595d24deec 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
@@ -18,35 +18,36 @@ declare double @div.double.value()
define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 {
; GFX7LESS-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_unsafe:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB0_3
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX7LESS-NEXT: v_mul_f32_e32 v2, 4.0, v0
-; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s2
+; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_subrev_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB0_2
; GFX7LESS-NEXT: .LBB0_3:
@@ -54,27 +55,28 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
;
; GFX9-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_unsafe:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s6, v0
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB0_3
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
-; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s4
; GFX9-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -86,27 +88,28 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
;
; GFX1064-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_unsafe:
; GFX1064: ; %bb.0:
-; GFX1064-NEXT: s_mov_b64 s[2:3], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-NEXT: v_readfirstlane_b32 s6, v0
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB0_3
; GFX1064-NEXT: ; %bb.1:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
-; GFX1064-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX1064-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v1, s4
+; GFX1064-NEXT: v_mov_b32_e32 v1, s2
+; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -118,61 +121,64 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
;
; GFX1032-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_unsafe:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: s_mov_b32 s3, exec_lo
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s3, 0
+; GFX1032-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB0_3
; GFX1032-NEXT: ; %bb.1:
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
-; GFX1032-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB0_2
; GFX1032-NEXT: .LBB0_3:
; GFX1032-NEXT: s_endpgm
;
; GFX1164-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_unsafe:
; GFX1164: ; %bb.0:
-; GFX1164-NEXT: s_mov_b64 s[2:3], exec
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB0_3
; GFX1164-NEXT: ; %bb.1:
; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1164-NEXT: v_mov_b32_e32 v3, 0
-; GFX1164-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1164-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_load_b32 s4, s[0:1], 0x0
+; GFX1164-NEXT: s_load_b32 s2, s[0:1], 0x0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mov_b32_e32 v1, s4
+; GFX1164-NEXT: v_mov_b32_e32 v1, s2
+; GFX1164-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-NEXT: v_mov_b32_e32 v1, v0
@@ -185,69 +191,71 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
;
; GFX1132-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_unsafe:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_mov_b32 s3, exec_lo
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-NEXT: s_mov_b32 s3, 0
+; GFX1132-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB0_3
; GFX1132-NEXT: ; %bb.1:
; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1132-NEXT: v_mov_b32_e32 v3, 0
-; GFX1132-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
+; GFX1132-NEXT: v_mov_b32_e32 v2, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_load_b32 s4, s[0:1], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_dual_mul_f32 v2, 4.0, v0 :: v_dual_mov_b32 v1, s4
+; GFX1132-NEXT: v_mov_b32_e32 v1, s4
; GFX1132-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-NEXT: s_cbranch_execnz .LBB0_2
; GFX1132-NEXT: .LBB0_3:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_unsafe:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB0_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
-; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, s6
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, s2
+; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
; GFX7LESS-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX7LESS-DPP-NEXT: v_subrev_f32_e32 v0, s6, v1
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB0_2
; GFX7LESS-DPP-NEXT: .LBB0_3:
@@ -255,27 +263,28 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
;
; GFX9-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_unsafe:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s6, v0
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB0_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
-; GFX9-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX9-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -287,27 +296,28 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
;
; GFX1064-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_unsafe:
; GFX1064-DPP: ; %bb.0:
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s6, v0
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB0_3
; GFX1064-DPP-NEXT: ; %bb.1:
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX1064-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX1064-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s4
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s2
+; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -319,61 +329,64 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
;
; GFX1032-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_unsafe:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: s_mov_b32 s3, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
+; GFX1032-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s2, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB0_3
; GFX1032-DPP-NEXT: ; %bb.1:
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
-; GFX1032-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB0_2
; GFX1032-DPP-NEXT: .LBB0_3:
; GFX1032-DPP-NEXT: s_endpgm
;
; GFX1164-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_unsafe:
; GFX1164-DPP: ; %bb.0:
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB0_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1164-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: s_load_b32 s4, s[0:1], 0x0
+; GFX1164-DPP-NEXT: s_load_b32 s2, s[0:1], 0x0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s4
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s2
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-DPP-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -386,34 +399,35 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
;
; GFX1132-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_unsafe:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_mov_b32 s3, exec_lo
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s3, 0
+; GFX1132-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB0_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1132-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_load_b32 s4, s[0:1], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_dual_mul_f32 v2, 4.0, v0 :: v_dual_mov_b32 v1, s4
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1132-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-DPP-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB0_2
; GFX1132-DPP-NEXT: .LBB0_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -453,24 +467,22 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX7LESS-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_brev_b32 s6, 1
+; GFX7LESS-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s2
-; GFX7LESS-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f32_e32 v2, s4, v2
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB1_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: v_readlane_b32 s3, v0, s2
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s3
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX7LESS-NEXT: v_add_f32_e32 v2, s6, v2
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v2
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB1_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
@@ -482,15 +494,15 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX7LESS-NEXT: .LBB1_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_subrev_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB1_4
; GFX7LESS-NEXT: .LBB1_5:
@@ -526,34 +538,34 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX9-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_brev_b32 s4, 1
+; GFX9-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s4, v0, s2
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_readlane_b32 s3, v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: s_bitset0_b64 s[0:1], s2
; GFX9-NEXT: v_add_f32_e32 v2, s4, v2
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9-NEXT: s_cbranch_execz .LBB1_5
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX9-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-NEXT: .LBB1_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -593,34 +605,33 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_brev_b32 s4, 1
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1064-NEXT: v_add_f32_e32 v2, s3, v2
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX1064-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1064-NEXT: v_add_f32_e64 v2, s4, s3
+; GFX1064-NEXT: v_readfirstlane_b32 s4, v2
; GFX1064-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064-NEXT: s_cbranch_execz .LBB1_5
; GFX1064-NEXT: ; %bb.3:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1064-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-NEXT: .LBB1_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -660,38 +671,37 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_brev_b32 s2, 1
+; GFX1032-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f32_e32 v2, s2, v2
+; GFX1032-NEXT: v_readlane_b32 s3, v0, s1
+; GFX1032-NEXT: s_bitset0_b32 s0, s1
+; GFX1032-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1032-NEXT: v_add_f32_e64 v2, s2, s3
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v2
; GFX1032-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1032-NEXT: s_cbranch_execz .LBB1_5
; GFX1032-NEXT: ; %bb.3:
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
+; GFX1032-NEXT: s_mov_b32 s3, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1032-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-NEXT: .LBB1_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB1_4
; GFX1032-NEXT: .LBB1_5:
; GFX1032-NEXT: s_endpgm
@@ -716,37 +726,36 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_brev_b32 s4, 1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f32_e32 v2, s3, v2
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: v_add_f32_e64 v2, s4, s3
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_readfirstlane_b32 s4, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: ; %bb.2:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB1_5
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v3, 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b32 v1, v3, s[0:1]
+; GFX1164-NEXT: global_load_b32 v1, v2, s[0:1]
; GFX1164-NEXT: .LBB1_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-NEXT: v_mov_b32_e32 v1, v0
@@ -778,42 +787,40 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: s_brev_b32 s2, 1
+; GFX1132-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f32_e32 v2, s2, v2
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: v_readlane_b32 s3, v0, s1
+; GFX1132-NEXT: s_bitset0_b32 s0, s1
+; GFX1132-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1132-NEXT: v_add_f32_e64 v2, s2, s3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: v_readfirstlane_b32 s2, v2
; GFX1132-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
+; GFX1132-NEXT: ; %bb.2:
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB1_5
; GFX1132-NEXT: ; %bb.3:
; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v3, 0
+; GFX1132-NEXT: v_mov_b32_e32 v2, 0
+; GFX1132-NEXT: s_mov_b32 s3, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b32 v1, v3, s[0:1]
+; GFX1132-NEXT: global_load_b32 v1, v2, s[0:1]
; GFX1132-NEXT: .LBB1_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-NEXT: s_cbranch_execnz .LBB1_4
; GFX1132-NEXT: .LBB1_5:
; GFX1132-NEXT: s_endpgm
@@ -904,30 +911,19 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[0:1]
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v4 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v3, v4, v3
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
@@ -984,44 +980,42 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1064-DPP-NEXT: ds_permute_b32 v4, v5, v3
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v3, 32
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_add_f32_e64 v3, s2, s3
+; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB1_3
; GFX1064-DPP-NEXT: ; %bb.1:
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-DPP-NEXT: .LBB1_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -1062,43 +1056,37 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s0
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX1032-DPP-NEXT: v_readlane_b32 s2, v3, 31
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB1_3
; GFX1032-DPP-NEXT: ; %bb.1:
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-DPP-NEXT: .LBB1_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB1_2
; GFX1032-DPP-NEXT: .LBB1_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -1124,52 +1112,48 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s[0:1]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1164-DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v2, v1
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: v_readlane_b32 s4, v1, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB1_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b32 v5, v6, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b32 v5, v0, s[0:1]
; GFX1164-DPP-NEXT: .LBB1_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_sub_f32_e32 v4, v5, v0
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v4, v6, v[4:5], s[0:1] glc
+; GFX1164-DPP-NEXT: v_subrev_f32_e32 v4, s4, v5
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v4, v0, v[4:5], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v4
@@ -1202,51 +1186,41 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s3, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB1_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_load_b32 v5, v6, s[0:1]
+; GFX1132-DPP-NEXT: global_load_b32 v4, v0, s[0:1]
; GFX1132-DPP-NEXT: .LBB1_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_sub_f32_e32 v4, v5, v0
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v4, v6, v[4:5], s[0:1] glc
+; GFX1132-DPP-NEXT: v_subrev_f32_e32 v3, s2, v4
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v3, v0, v[3:4], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v5
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v5, v4
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, v3
+; GFX1132-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB1_2
; GFX1132-DPP-NEXT: .LBB1_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -1258,45 +1232,36 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp(ptr addrspace(1) %ptr) #1 {
; GFX7LESS-LABEL: global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT: s_mov_b32 s14, -1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0
-; GFX7LESS-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0
-; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB2_3
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dword s2, s[0:1], 0x0
-; GFX7LESS-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v1, s2
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_subrev_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB2_2
; GFX7LESS-NEXT: .LBB2_3:
@@ -1304,37 +1269,28 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_one_as_scope
;
; GFX9-LABEL: global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s14, -1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-NEXT: s_add_u32 s12, s12, s11
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s6, v0
+; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB2_3
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX9-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -1347,34 +1303,27 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_one_as_scope
; GFX1064-LABEL: global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT: s_mov_b32 s14, -1
-; GFX1064-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB2_3
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1064-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -1387,75 +1336,63 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_one_as_scope
; GFX1032-LABEL: global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1032: ; %bb.0:
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT: s_mov_b32 s14, -1
-; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s3, 0
+; GFX1032-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB2_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mov_b32_e32 v1, s3
-; GFX1032-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB2_2
; GFX1032-NEXT: .LBB2_3:
; GFX1032-NEXT: s_endpgm
;
; GFX1164-LABEL: global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1164: ; %bb.0:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_clause 0x1
-; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB2_3
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v3, 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_load_b32 s2, s[0:1], 0x0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX1164-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: v_mov_b32_e32 v1, s2
; GFX1164-NEXT: s_mov_b64 s[2:3], 0
-; GFX1164-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1164-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-NEXT: v_mov_b32_e32 v1, v0
@@ -1468,85 +1405,71 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_one_as_scope
;
; GFX1132-LABEL: global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_clause 0x1
-; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-NEXT: s_mov_b32 s3, 0
+; GFX1132-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB2_3
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v3, 0
+; GFX1132-NEXT: v_mov_b32_e32 v2, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b32 s3, s[0:1], 0x0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1132-NEXT: s_load_b32 s4, s[0:1], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mul_f32 v2, 4.0, v0
+; GFX1132-NEXT: v_mov_b32_e32 v1, s4
; GFX1132-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-NEXT: s_cbranch_execnz .LBB2_2
; GFX1132-NEXT: .LBB2_3:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT: s_mov_b32 s14, -1
; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-DPP-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0
-; GFX7LESS-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0
-; GFX7LESS-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB2_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, s2
; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
; GFX7LESS-DPP-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX7LESS-DPP-NEXT: v_subrev_f32_e32 v0, s6, v1
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB2_2
; GFX7LESS-DPP-NEXT: .LBB2_3:
@@ -1554,37 +1477,28 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_one_as_scope
;
; GFX9-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT: s_mov_b32 s14, -1
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-DPP-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB2_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX9-DPP-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -1597,34 +1511,27 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_one_as_scope
; GFX1064-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1064-DPP: ; %bb.0:
; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1064-DPP-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB2_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1064-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -1637,75 +1544,63 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_one_as_scope
; GFX1032-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1032-DPP: ; %bb.0:
; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
+; GFX1032-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB2_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s3
-; GFX1032-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-DPP-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB2_2
; GFX1032-DPP-NEXT: .LBB2_3:
; GFX1032-DPP-NEXT: s_endpgm
;
; GFX1164-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1164-DPP: ; %bb.0:
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_clause 0x1
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB2_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_load_b32 s2, s[0:1], 0x0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s2
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX1164-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1164-DPP-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-DPP-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -1718,40 +1613,35 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_one_as_scope
;
; GFX1132-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_clause 0x1
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s3, 0
+; GFX1132-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB2_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_load_b32 s3, s[0:1], 0x0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1132-DPP-NEXT: s_load_b32 s4, s[0:1], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mul_f32 v2, 4.0, v0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1132-DPP-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-DPP-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB2_2
; GFX1132-DPP-NEXT: .LBB2_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -1792,24 +1682,22 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX7LESS-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_brev_b32 s6, 1
+; GFX7LESS-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s2
-; GFX7LESS-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f32_e32 v2, s4, v2
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB3_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: v_readlane_b32 s3, v0, s2
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s3
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX7LESS-NEXT: v_add_f32_e32 v2, s6, v2
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v2
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB3_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB3_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
@@ -1821,15 +1709,15 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX7LESS-NEXT: .LBB3_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_subrev_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB3_4
; GFX7LESS-NEXT: .LBB3_5:
@@ -1865,34 +1753,34 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX9-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_brev_b32 s4, 1
+; GFX9-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s4, v0, s2
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_readlane_b32 s3, v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: s_bitset0_b64 s[0:1], s2
; GFX9-NEXT: v_add_f32_e32 v2, s4, v2
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9-NEXT: s_cbranch_execz .LBB3_5
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX9-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-NEXT: .LBB3_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -1932,34 +1820,33 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_brev_b32 s4, 1
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1064-NEXT: v_add_f32_e32 v2, s3, v2
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX1064-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1064-NEXT: v_add_f32_e64 v2, s4, s3
+; GFX1064-NEXT: v_readfirstlane_b32 s4, v2
; GFX1064-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064-NEXT: s_cbranch_execz .LBB3_5
; GFX1064-NEXT: ; %bb.3:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1064-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-NEXT: .LBB3_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -1999,38 +1886,37 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_brev_b32 s2, 1
+; GFX1032-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f32_e32 v2, s2, v2
+; GFX1032-NEXT: v_readlane_b32 s3, v0, s1
+; GFX1032-NEXT: s_bitset0_b32 s0, s1
+; GFX1032-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1032-NEXT: v_add_f32_e64 v2, s2, s3
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v2
; GFX1032-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1032-NEXT: s_cbranch_execz .LBB3_5
; GFX1032-NEXT: ; %bb.3:
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
+; GFX1032-NEXT: s_mov_b32 s3, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1032-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-NEXT: .LBB3_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB3_4
; GFX1032-NEXT: .LBB3_5:
; GFX1032-NEXT: s_endpgm
@@ -2055,37 +1941,36 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_brev_b32 s4, 1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f32_e32 v2, s3, v2
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: v_add_f32_e64 v2, s4, s3
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_readfirstlane_b32 s4, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: ; %bb.2:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB3_5
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v3, 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b32 v1, v3, s[0:1]
+; GFX1164-NEXT: global_load_b32 v1, v2, s[0:1]
; GFX1164-NEXT: .LBB3_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-NEXT: v_mov_b32_e32 v1, v0
@@ -2117,42 +2002,40 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: s_brev_b32 s2, 1
+; GFX1132-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f32_e32 v2, s2, v2
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: v_readlane_b32 s3, v0, s1
+; GFX1132-NEXT: s_bitset0_b32 s0, s1
+; GFX1132-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1132-NEXT: v_add_f32_e64 v2, s2, s3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: v_readfirstlane_b32 s2, v2
; GFX1132-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
+; GFX1132-NEXT: ; %bb.2:
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB3_5
; GFX1132-NEXT: ; %bb.3:
; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v3, 0
+; GFX1132-NEXT: v_mov_b32_e32 v2, 0
+; GFX1132-NEXT: s_mov_b32 s3, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b32 v1, v3, s[0:1]
+; GFX1132-NEXT: global_load_b32 v1, v2, s[0:1]
; GFX1132-NEXT: .LBB3_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-NEXT: s_cbranch_execnz .LBB3_4
; GFX1132-NEXT: .LBB3_5:
; GFX1132-NEXT: s_endpgm
@@ -2243,30 +2126,19 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[0:1]
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v4 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v3, v4, v3
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
@@ -2323,44 +2195,42 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1064-DPP-NEXT: ds_permute_b32 v4, v5, v3
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v3, 32
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_add_f32_e64 v3, s2, s3
+; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB3_3
; GFX1064-DPP-NEXT: ; %bb.1:
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-DPP-NEXT: .LBB3_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -2401,43 +2271,37 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s0
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX1032-DPP-NEXT: v_readlane_b32 s2, v3, 31
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB3_3
; GFX1032-DPP-NEXT: ; %bb.1:
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-DPP-NEXT: .LBB3_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB3_2
; GFX1032-DPP-NEXT: .LBB3_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -2463,52 +2327,48 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s[0:1]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1164-DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v2, v1
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: v_readlane_b32 s4, v1, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB3_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b32 v5, v6, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b32 v5, v0, s[0:1]
; GFX1164-DPP-NEXT: .LBB3_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_sub_f32_e32 v4, v5, v0
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v4, v6, v[4:5], s[0:1] glc
+; GFX1164-DPP-NEXT: v_subrev_f32_e32 v4, s4, v5
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v4, v0, v[4:5], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v4
@@ -2541,51 +2401,41 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s3, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB3_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_load_b32 v5, v6, s[0:1]
+; GFX1132-DPP-NEXT: global_load_b32 v4, v0, s[0:1]
; GFX1132-DPP-NEXT: .LBB3_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_sub_f32_e32 v4, v5, v0
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v4, v6, v[4:5], s[0:1] glc
+; GFX1132-DPP-NEXT: v_subrev_f32_e32 v3, s2, v4
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v3, v0, v[3:4], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v5
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v5, v4
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, v3
+; GFX1132-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB3_2
; GFX1132-DPP-NEXT: .LBB3_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -2597,45 +2447,36 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp(ptr addrspace(1) %ptr) #2{
; GFX7LESS-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT: s_mov_b32 s14, -1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0
-; GFX7LESS-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0
-; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB4_3
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dword s2, s[0:1], 0x0
-; GFX7LESS-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v1, s2
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_subrev_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB4_2
; GFX7LESS-NEXT: .LBB4_3:
@@ -2643,37 +2484,28 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
;
; GFX9-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s14, -1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-NEXT: s_add_u32 s12, s12, s11
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s6, v0
+; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB4_3
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX9-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -2686,34 +2518,27 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
; GFX1064-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp:
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT: s_mov_b32 s14, -1
-; GFX1064-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB4_3
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1064-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -2726,75 +2551,63 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
; GFX1032-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp:
; GFX1032: ; %bb.0:
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT: s_mov_b32 s14, -1
-; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s3, 0
+; GFX1032-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB4_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mov_b32_e32 v1, s3
-; GFX1032-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB4_2
; GFX1032-NEXT: .LBB4_3:
; GFX1032-NEXT: s_endpgm
;
; GFX1164-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp:
; GFX1164: ; %bb.0:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_clause 0x1
-; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB4_3
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v3, 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_load_b32 s2, s[0:1], 0x0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX1164-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: v_mov_b32_e32 v1, s2
; GFX1164-NEXT: s_mov_b64 s[2:3], 0
-; GFX1164-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1164-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-NEXT: v_mov_b32_e32 v1, v0
@@ -2807,85 +2620,71 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
;
; GFX1132-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_clause 0x1
-; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-NEXT: s_mov_b32 s3, 0
+; GFX1132-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB4_3
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v3, 0
+; GFX1132-NEXT: v_mov_b32_e32 v2, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b32 s3, s[0:1], 0x0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1132-NEXT: s_load_b32 s4, s[0:1], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mul_f32 v2, 4.0, v0
+; GFX1132-NEXT: v_mov_b32_e32 v1, s4
; GFX1132-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-NEXT: s_cbranch_execnz .LBB4_2
; GFX1132-NEXT: .LBB4_3:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT: s_mov_b32 s14, -1
; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-DPP-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0
-; GFX7LESS-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0
-; GFX7LESS-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB4_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, s2
; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
; GFX7LESS-DPP-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX7LESS-DPP-NEXT: v_subrev_f32_e32 v0, s6, v1
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB4_2
; GFX7LESS-DPP-NEXT: .LBB4_3:
@@ -2893,37 +2692,28 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
;
; GFX9-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT: s_mov_b32 s14, -1
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-DPP-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB4_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX9-DPP-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -2936,34 +2726,27 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
; GFX1064-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp:
; GFX1064-DPP: ; %bb.0:
; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1064-DPP-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB4_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1064-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -2976,75 +2759,63 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
; GFX1032-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp:
; GFX1032-DPP: ; %bb.0:
; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
+; GFX1032-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB4_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s3
-; GFX1032-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-DPP-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB4_2
; GFX1032-DPP-NEXT: .LBB4_3:
; GFX1032-DPP-NEXT: s_endpgm
;
; GFX1164-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp:
; GFX1164-DPP: ; %bb.0:
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_clause 0x1
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB4_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_load_b32 s2, s[0:1], 0x0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s2
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX1164-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1164-DPP-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-DPP-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -3057,40 +2828,35 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
;
; GFX1132-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_clause 0x1
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s3, 0
+; GFX1132-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB4_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_load_b32 s3, s[0:1], 0x0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1132-DPP-NEXT: s_load_b32 s4, s[0:1], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mul_f32 v2, 4.0, v0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1132-DPP-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-DPP-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB4_2
; GFX1132-DPP-NEXT: .LBB4_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -3131,24 +2897,22 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX7LESS-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_brev_b32 s6, 1
+; GFX7LESS-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s2
-; GFX7LESS-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f32_e32 v2, s4, v2
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB5_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: v_readlane_b32 s3, v0, s2
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s3
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX7LESS-NEXT: v_add_f32_e32 v2, s6, v2
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v2
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB5_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB5_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
@@ -3160,15 +2924,15 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX7LESS-NEXT: .LBB5_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_subrev_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB5_4
; GFX7LESS-NEXT: .LBB5_5:
@@ -3204,34 +2968,34 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX9-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_brev_b32 s4, 1
+; GFX9-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s4, v0, s2
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_readlane_b32 s3, v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: s_bitset0_b64 s[0:1], s2
; GFX9-NEXT: v_add_f32_e32 v2, s4, v2
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9-NEXT: s_cbranch_execz .LBB5_5
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX9-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-NEXT: .LBB5_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -3271,34 +3035,33 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_brev_b32 s4, 1
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1064-NEXT: v_add_f32_e32 v2, s3, v2
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX1064-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1064-NEXT: v_add_f32_e64 v2, s4, s3
+; GFX1064-NEXT: v_readfirstlane_b32 s4, v2
; GFX1064-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064-NEXT: s_cbranch_execz .LBB5_5
; GFX1064-NEXT: ; %bb.3:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1064-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-NEXT: .LBB5_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -3338,38 +3101,37 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_brev_b32 s2, 1
+; GFX1032-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f32_e32 v2, s2, v2
+; GFX1032-NEXT: v_readlane_b32 s3, v0, s1
+; GFX1032-NEXT: s_bitset0_b32 s0, s1
+; GFX1032-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1032-NEXT: v_add_f32_e64 v2, s2, s3
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v2
; GFX1032-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1032-NEXT: s_cbranch_execz .LBB5_5
; GFX1032-NEXT: ; %bb.3:
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
+; GFX1032-NEXT: s_mov_b32 s3, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1032-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-NEXT: .LBB5_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB5_4
; GFX1032-NEXT: .LBB5_5:
; GFX1032-NEXT: s_endpgm
@@ -3394,37 +3156,36 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_brev_b32 s4, 1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f32_e32 v2, s3, v2
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: v_add_f32_e64 v2, s4, s3
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_readfirstlane_b32 s4, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: ; %bb.2:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB5_5
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v3, 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b32 v1, v3, s[0:1]
+; GFX1164-NEXT: global_load_b32 v1, v2, s[0:1]
; GFX1164-NEXT: .LBB5_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-NEXT: v_mov_b32_e32 v1, v0
@@ -3456,42 +3217,40 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: s_brev_b32 s2, 1
+; GFX1132-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f32_e32 v2, s2, v2
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: v_readlane_b32 s3, v0, s1
+; GFX1132-NEXT: s_bitset0_b32 s0, s1
+; GFX1132-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1132-NEXT: v_add_f32_e64 v2, s2, s3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: v_readfirstlane_b32 s2, v2
; GFX1132-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
+; GFX1132-NEXT: ; %bb.2:
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB5_5
; GFX1132-NEXT: ; %bb.3:
; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v3, 0
+; GFX1132-NEXT: v_mov_b32_e32 v2, 0
+; GFX1132-NEXT: s_mov_b32 s3, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b32 v1, v3, s[0:1]
+; GFX1132-NEXT: global_load_b32 v1, v2, s[0:1]
; GFX1132-NEXT: .LBB5_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-NEXT: s_cbranch_execnz .LBB5_4
; GFX1132-NEXT: .LBB5_5:
; GFX1132-NEXT: s_endpgm
@@ -3582,30 +3341,19 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[0:1]
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v4 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v3, v4, v3
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
@@ -3662,44 +3410,42 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1064-DPP-NEXT: ds_permute_b32 v4, v5, v3
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v3, 32
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_add_f32_e64 v3, s2, s3
+; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB5_3
; GFX1064-DPP-NEXT: ; %bb.1:
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-DPP-NEXT: .LBB5_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -3740,43 +3486,37 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s0
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX1032-DPP-NEXT: v_readlane_b32 s2, v3, 31
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB5_3
; GFX1032-DPP-NEXT: ; %bb.1:
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-DPP-NEXT: .LBB5_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB5_2
; GFX1032-DPP-NEXT: .LBB5_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -3802,52 +3542,48 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s[0:1]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1164-DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v2, v1
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: v_readlane_b32 s4, v1, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB5_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b32 v5, v6, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b32 v5, v0, s[0:1]
; GFX1164-DPP-NEXT: .LBB5_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_sub_f32_e32 v4, v5, v0
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v4, v6, v[4:5], s[0:1] glc
+; GFX1164-DPP-NEXT: v_subrev_f32_e32 v4, s4, v5
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v4, v0, v[4:5], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v4
@@ -3880,51 +3616,41 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s3, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB5_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_load_b32 v5, v6, s[0:1]
+; GFX1132-DPP-NEXT: global_load_b32 v4, v0, s[0:1]
; GFX1132-DPP-NEXT: .LBB5_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_sub_f32_e32 v4, v5, v0
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v4, v6, v[4:5], s[0:1] glc
+; GFX1132-DPP-NEXT: v_subrev_f32_e32 v3, s2, v4
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v3, v0, v[3:4], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v5
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v5, v4
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, v3
+; GFX1132-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB5_2
; GFX1132-DPP-NEXT: .LBB5_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -3966,24 +3692,22 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX7LESS-NEXT: .LBB6_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_brev_b32 s6, 1
+; GFX7LESS-NEXT: .LBB6_1: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s2
-; GFX7LESS-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f32_e32 v2, s4, v2
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB6_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: v_readlane_b32 s3, v0, s2
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s3
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX7LESS-NEXT: v_add_f32_e32 v2, s6, v2
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v2
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB6_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB6_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
@@ -3995,15 +3719,15 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX7LESS-NEXT: .LBB6_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_subrev_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB6_4
; GFX7LESS-NEXT: .LBB6_5:
@@ -4039,34 +3763,34 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX9-NEXT: .LBB6_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_brev_b32 s4, 1
+; GFX9-NEXT: .LBB6_1: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s4, v0, s2
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_readlane_b32 s3, v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: s_bitset0_b64 s[0:1], s2
; GFX9-NEXT: v_add_f32_e32 v2, s4, v2
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: s_cbranch_scc1 .LBB6_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9-NEXT: s_cbranch_execz .LBB6_5
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX9-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-NEXT: .LBB6_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -4106,34 +3830,33 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB6_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_brev_b32 s4, 1
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: .LBB6_1: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1064-NEXT: v_add_f32_e32 v2, s3, v2
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX1064-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1064-NEXT: v_add_f32_e64 v2, s4, s3
+; GFX1064-NEXT: v_readfirstlane_b32 s4, v2
; GFX1064-NEXT: s_cbranch_scc1 .LBB6_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064-NEXT: s_cbranch_execz .LBB6_5
; GFX1064-NEXT: ; %bb.3:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1064-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-NEXT: .LBB6_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -4173,38 +3896,37 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB6_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_brev_b32 s2, 1
+; GFX1032-NEXT: .LBB6_1: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f32_e32 v2, s2, v2
+; GFX1032-NEXT: v_readlane_b32 s3, v0, s1
+; GFX1032-NEXT: s_bitset0_b32 s0, s1
+; GFX1032-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1032-NEXT: v_add_f32_e64 v2, s2, s3
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v2
; GFX1032-NEXT: s_cbranch_scc1 .LBB6_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1032-NEXT: s_cbranch_execz .LBB6_5
; GFX1032-NEXT: ; %bb.3:
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
+; GFX1032-NEXT: s_mov_b32 s3, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1032-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-NEXT: .LBB6_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB6_4
; GFX1032-NEXT: .LBB6_5:
; GFX1032-NEXT: s_endpgm
@@ -4229,37 +3951,36 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB6_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_brev_b32 s4, 1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: .LBB6_1: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f32_e32 v2, s3, v2
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: v_add_f32_e64 v2, s4, s3
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_readfirstlane_b32 s4, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB6_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: ; %bb.2:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB6_5
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v3, 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b32 v1, v3, s[0:1]
+; GFX1164-NEXT: global_load_b32 v1, v2, s[0:1]
; GFX1164-NEXT: .LBB6_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-NEXT: v_mov_b32_e32 v1, v0
@@ -4291,42 +4012,40 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB6_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: s_brev_b32 s2, 1
+; GFX1132-NEXT: .LBB6_1: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f32_e32 v2, s2, v2
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: v_readlane_b32 s3, v0, s1
+; GFX1132-NEXT: s_bitset0_b32 s0, s1
+; GFX1132-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1132-NEXT: v_add_f32_e64 v2, s2, s3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: v_readfirstlane_b32 s2, v2
; GFX1132-NEXT: s_cbranch_scc1 .LBB6_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
+; GFX1132-NEXT: ; %bb.2:
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB6_5
; GFX1132-NEXT: ; %bb.3:
; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v3, 0
+; GFX1132-NEXT: v_mov_b32_e32 v2, 0
+; GFX1132-NEXT: s_mov_b32 s3, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b32 v1, v3, s[0:1]
+; GFX1132-NEXT: global_load_b32 v1, v2, s[0:1]
; GFX1132-NEXT: .LBB6_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-NEXT: s_cbranch_execnz .LBB6_4
; GFX1132-NEXT: .LBB6_5:
; GFX1132-NEXT: s_endpgm
@@ -4417,30 +4136,19 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[0:1]
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v4 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v3, v4, v3
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
@@ -4497,44 +4205,42 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1064-DPP-NEXT: ds_permute_b32 v4, v5, v3
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v3, 32
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_add_f32_e64 v3, s2, s3
+; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB6_3
; GFX1064-DPP-NEXT: ; %bb.1:
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-DPP-NEXT: .LBB6_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -4575,43 +4281,37 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s0
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX1032-DPP-NEXT: v_readlane_b32 s2, v3, 31
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB6_3
; GFX1032-DPP-NEXT: ; %bb.1:
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-DPP-NEXT: .LBB6_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB6_2
; GFX1032-DPP-NEXT: .LBB6_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -4637,52 +4337,48 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s[0:1]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1164-DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v2, v1
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: v_readlane_b32 s4, v1, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB6_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b32 v5, v6, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b32 v5, v0, s[0:1]
; GFX1164-DPP-NEXT: .LBB6_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_sub_f32_e32 v4, v5, v0
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v4, v6, v[4:5], s[0:1] glc
+; GFX1164-DPP-NEXT: v_subrev_f32_e32 v4, s4, v5
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v4, v0, v[4:5], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v4
@@ -4715,51 +4411,41 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s3, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB6_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_load_b32 v5, v6, s[0:1]
+; GFX1132-DPP-NEXT: global_load_b32 v4, v0, s[0:1]
; GFX1132-DPP-NEXT: .LBB6_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_sub_f32_e32 v4, v5, v0
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v4, v6, v[4:5], s[0:1] glc
+; GFX1132-DPP-NEXT: v_subrev_f32_e32 v3, s2, v4
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v3, v0, v[3:4], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v5
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v5, v4
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, v3
+; GFX1132-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB6_2
; GFX1132-DPP-NEXT: .LBB6_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -4771,45 +4457,36 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_default_scope_strictfp(ptr addrspace(1) %ptr) #2 {
; GFX7LESS-LABEL: global_atomic_fsub_uni_address_uni_value_default_scope_strictfp:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT: s_mov_b32 s14, -1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0
-; GFX7LESS-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0
-; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB7_3
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dword s2, s[0:1], 0x0
-; GFX7LESS-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v1, s2
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_subrev_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB7_2
; GFX7LESS-NEXT: .LBB7_3:
@@ -4817,37 +4494,28 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_default_scop
;
; GFX9-LABEL: global_atomic_fsub_uni_address_uni_value_default_scope_strictfp:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s14, -1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-NEXT: s_add_u32 s12, s12, s11
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s6, v0
+; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB7_3
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX9-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -4860,34 +4528,27 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_default_scop
; GFX1064-LABEL: global_atomic_fsub_uni_address_uni_value_default_scope_strictfp:
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT: s_mov_b32 s14, -1
-; GFX1064-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB7_3
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1064-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -4900,75 +4561,63 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_default_scop
; GFX1032-LABEL: global_atomic_fsub_uni_address_uni_value_default_scope_strictfp:
; GFX1032: ; %bb.0:
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT: s_mov_b32 s14, -1
-; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s3, 0
+; GFX1032-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB7_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mov_b32_e32 v1, s3
-; GFX1032-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB7_2
; GFX1032-NEXT: .LBB7_3:
; GFX1032-NEXT: s_endpgm
;
; GFX1164-LABEL: global_atomic_fsub_uni_address_uni_value_default_scope_strictfp:
; GFX1164: ; %bb.0:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_clause 0x1
-; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB7_3
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v3, 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_load_b32 s2, s[0:1], 0x0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX1164-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: v_mov_b32_e32 v1, s2
; GFX1164-NEXT: s_mov_b64 s[2:3], 0
-; GFX1164-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1164-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-NEXT: v_mov_b32_e32 v1, v0
@@ -4981,85 +4630,71 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_default_scop
;
; GFX1132-LABEL: global_atomic_fsub_uni_address_uni_value_default_scope_strictfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_clause 0x1
-; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-NEXT: s_mov_b32 s3, 0
+; GFX1132-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB7_3
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v3, 0
+; GFX1132-NEXT: v_mov_b32_e32 v2, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b32 s3, s[0:1], 0x0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1132-NEXT: s_load_b32 s4, s[0:1], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mul_f32 v2, 4.0, v0
+; GFX1132-NEXT: v_mov_b32_e32 v1, s4
; GFX1132-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-NEXT: s_cbranch_execnz .LBB7_2
; GFX1132-NEXT: .LBB7_3:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_default_scope_strictfp:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT: s_mov_b32 s14, -1
; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-DPP-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0
-; GFX7LESS-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0
-; GFX7LESS-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, s2
; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
; GFX7LESS-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX7LESS-DPP-NEXT: v_subrev_f32_e32 v0, s6, v1
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB7_2
; GFX7LESS-DPP-NEXT: .LBB7_3:
@@ -5067,37 +4702,28 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_default_scop
;
; GFX9-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_default_scope_strictfp:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT: s_mov_b32 s14, -1
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-DPP-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX9-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -5110,34 +4736,27 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_default_scop
; GFX1064-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_default_scope_strictfp:
; GFX1064-DPP: ; %bb.0:
; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1064-DPP-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1064-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -5150,75 +4769,63 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_default_scop
; GFX1032-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_default_scope_strictfp:
; GFX1032-DPP: ; %bb.0:
; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
+; GFX1032-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s3
-; GFX1032-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB7_2
; GFX1032-DPP-NEXT: .LBB7_3:
; GFX1032-DPP-NEXT: s_endpgm
;
; GFX1164-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_default_scope_strictfp:
; GFX1164-DPP: ; %bb.0:
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_clause 0x1
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_load_b32 s2, s[0:1], 0x0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s2
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX1164-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1164-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-DPP-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -5231,40 +4838,35 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_default_scop
;
; GFX1132-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_default_scope_strictfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_clause 0x1
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s3, 0
+; GFX1132-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_load_b32 s3, s[0:1], 0x0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1132-DPP-NEXT: s_load_b32 s4, s[0:1], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mul_f32 v2, 4.0, v0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1132-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-DPP-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB7_2
; GFX1132-DPP-NEXT: .LBB7_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -5304,24 +4906,22 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX7LESS-NEXT: .LBB8_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_brev_b32 s6, 1
+; GFX7LESS-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s2
-; GFX7LESS-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f32_e32 v2, s4, v2
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB8_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: v_readlane_b32 s3, v0, s2
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s3
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX7LESS-NEXT: v_add_f32_e32 v2, s6, v2
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v2
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB8_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB8_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
@@ -5333,15 +4933,15 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX7LESS-NEXT: .LBB8_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_subrev_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB8_4
; GFX7LESS-NEXT: .LBB8_5:
@@ -5377,34 +4977,34 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX9-NEXT: .LBB8_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_brev_b32 s4, 1
+; GFX9-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s4, v0, s2
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_readlane_b32 s3, v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: s_bitset0_b64 s[0:1], s2
; GFX9-NEXT: v_add_f32_e32 v2, s4, v2
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: s_cbranch_scc1 .LBB8_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9-NEXT: s_cbranch_execz .LBB8_5
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX9-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-NEXT: .LBB8_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -5444,34 +5044,33 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB8_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_brev_b32 s4, 1
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1064-NEXT: v_add_f32_e32 v2, s3, v2
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX1064-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1064-NEXT: v_add_f32_e64 v2, s4, s3
+; GFX1064-NEXT: v_readfirstlane_b32 s4, v2
; GFX1064-NEXT: s_cbranch_scc1 .LBB8_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064-NEXT: s_cbranch_execz .LBB8_5
; GFX1064-NEXT: ; %bb.3:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1064-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-NEXT: .LBB8_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -5511,38 +5110,37 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB8_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_brev_b32 s2, 1
+; GFX1032-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f32_e32 v2, s2, v2
+; GFX1032-NEXT: v_readlane_b32 s3, v0, s1
+; GFX1032-NEXT: s_bitset0_b32 s0, s1
+; GFX1032-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1032-NEXT: v_add_f32_e64 v2, s2, s3
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v2
; GFX1032-NEXT: s_cbranch_scc1 .LBB8_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1032-NEXT: s_cbranch_execz .LBB8_5
; GFX1032-NEXT: ; %bb.3:
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
+; GFX1032-NEXT: s_mov_b32 s3, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1032-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-NEXT: .LBB8_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB8_4
; GFX1032-NEXT: .LBB8_5:
; GFX1032-NEXT: s_endpgm
@@ -5567,37 +5165,36 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB8_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_brev_b32 s4, 1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f32_e32 v2, s3, v2
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: v_add_f32_e64 v2, s4, s3
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_readfirstlane_b32 s4, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB8_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: ; %bb.2:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB8_5
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v3, 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b32 v1, v3, s[0:1]
+; GFX1164-NEXT: global_load_b32 v1, v2, s[0:1]
; GFX1164-NEXT: .LBB8_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-NEXT: v_mov_b32_e32 v1, v0
@@ -5629,42 +5226,40 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB8_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: s_brev_b32 s2, 1
+; GFX1132-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f32_e32 v2, s2, v2
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: v_readlane_b32 s3, v0, s1
+; GFX1132-NEXT: s_bitset0_b32 s0, s1
+; GFX1132-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1132-NEXT: v_add_f32_e64 v2, s2, s3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: v_readfirstlane_b32 s2, v2
; GFX1132-NEXT: s_cbranch_scc1 .LBB8_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
+; GFX1132-NEXT: ; %bb.2:
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB8_5
; GFX1132-NEXT: ; %bb.3:
; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v3, 0
+; GFX1132-NEXT: v_mov_b32_e32 v2, 0
+; GFX1132-NEXT: s_mov_b32 s3, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b32 v1, v3, s[0:1]
+; GFX1132-NEXT: global_load_b32 v1, v2, s[0:1]
; GFX1132-NEXT: .LBB8_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-NEXT: s_cbranch_execnz .LBB8_4
; GFX1132-NEXT: .LBB8_5:
; GFX1132-NEXT: s_endpgm
@@ -5755,30 +5350,19 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[0:1]
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v4 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v3, v4, v3
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
@@ -5835,44 +5419,42 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1064-DPP-NEXT: ds_permute_b32 v4, v5, v3
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v3, 32
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_add_f32_e64 v3, s2, s3
+; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1064-DPP-NEXT: ; %bb.1:
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-DPP-NEXT: .LBB8_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -5913,43 +5495,37 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s0
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX1032-DPP-NEXT: v_readlane_b32 s2, v3, 31
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1032-DPP-NEXT: ; %bb.1:
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-DPP-NEXT: .LBB8_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB8_2
; GFX1032-DPP-NEXT: .LBB8_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -5975,52 +5551,48 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s[0:1]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1164-DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v2, v1
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: v_readlane_b32 s4, v1, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b32 v5, v6, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b32 v5, v0, s[0:1]
; GFX1164-DPP-NEXT: .LBB8_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_sub_f32_e32 v4, v5, v0
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v4, v6, v[4:5], s[0:1] glc
+; GFX1164-DPP-NEXT: v_subrev_f32_e32 v4, s4, v5
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v4, v0, v[4:5], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v4
@@ -6053,51 +5625,41 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s3, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_load_b32 v5, v6, s[0:1]
+; GFX1132-DPP-NEXT: global_load_b32 v4, v0, s[0:1]
; GFX1132-DPP-NEXT: .LBB8_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_sub_f32_e32 v4, v5, v0
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v4, v6, v[4:5], s[0:1] glc
+; GFX1132-DPP-NEXT: v_subrev_f32_e32 v3, s2, v4
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v3, v0, v[3:4], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v5
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v5, v4
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, v3
+; GFX1132-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB8_2
; GFX1132-DPP-NEXT: .LBB8_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -6109,39 +5671,41 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 {
; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s7, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB9_3
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-NEXT: v_cvt_f64_u32_e32 v[0:1], s2
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
-; GFX7LESS-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, s6
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, s7
+; GFX7LESS-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s8
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s9
; GFX7LESS-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -s[6:7]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB9_2
; GFX7LESS-NEXT: .LBB9_3:
@@ -6149,101 +5713,107 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
;
; GFX9-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s1, v1
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB9_3
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX9-NEXT: v_cvt_f64_u32_e32 v[0:1], s2
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NEXT: v_mov_b32_e32 v3, s5
+; GFX9-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
; GFX9-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB9_2
; GFX9-NEXT: .LBB9_3:
; GFX9-NEXT: s_endpgm
;
; GFX1064-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1064: ; %bb.0:
-; GFX1064-NEXT: s_mov_b64 s[2:3], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB9_3
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: v_cvt_f64_u32_e32 v[0:1], s2
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX1064-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v2, s2
-; GFX1064-NEXT: v_mov_b32_e32 v3, s3
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, s4
+; GFX1064-NEXT: v_mov_b32_e32 v3, s5
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB9_2
; GFX1064-NEXT: .LBB9_3:
; GFX1064-NEXT: s_endpgm
;
; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: s_mov_b32 s3, exec_lo
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s6, 0
+; GFX1032-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB9_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_cvt_f64_u32_e32 v[0:1], s3
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX1032-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v2, s4
; GFX1032-NEXT: v_mov_b32_e32 v3, s5
; GFX1032-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX1032-NEXT: s_cbranch_execnz .LBB9_2
; GFX1032-NEXT: .LBB9_3:
; GFX1032-NEXT: s_endpgm
@@ -6251,37 +5821,40 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
; GFX1164-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1164: ; %bb.0:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
; GFX1164-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1164-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB9_3
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX1164-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164-NEXT: v_mov_b32_e32 v3, s3
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, s4
+; GFX1164-NEXT: v_mov_b32_e32 v3, s5
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB9_2
; GFX1164-NEXT: .LBB9_3:
; GFX1164-NEXT: s_endpgm
@@ -6289,73 +5862,77 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
; GFX1132-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1132: ; %bb.0:
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1132-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-NEXT: s_mov_b32 s6, 0
+; GFX1132-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB9_3
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
-; GFX1132-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX1132-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX1132-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s6, vcc_lo, s6
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
; GFX1132-NEXT: s_cbranch_execnz .LBB9_2
; GFX1132-NEXT: .LBB9_3:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s7, v1
; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s2
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, s6
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, s7
+; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, s8
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, s9
; GFX7LESS-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[6:7]
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v6
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v4
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v7
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v5
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB9_2
; GFX7LESS-DPP-NEXT: .LBB9_3:
@@ -6363,101 +5940,107 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
;
; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX9-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s2
-; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s7
; GFX9-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB9_2
; GFX9-DPP-NEXT: .LBB9_3:
; GFX9-DPP-NEXT: s_endpgm
;
; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1064-DPP: ; %bb.0:
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s2
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX1064-DPP-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s2
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s3
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s4
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB9_2
; GFX1064-DPP-NEXT: .LBB9_3:
; GFX1064-DPP-NEXT: s_endpgm
;
; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: s_mov_b32 s3, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s6, 0
+; GFX1032-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s4
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s5
; GFX1032-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB9_2
; GFX1032-DPP-NEXT: .LBB9_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -6465,37 +6048,40 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1164-DPP: ; %bb.0:
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1164-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX1164-DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s3
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s4
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB9_2
; GFX1164-DPP-NEXT: .LBB9_3:
; GFX1164-DPP-NEXT: s_endpgm
@@ -6503,34 +6089,36 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1132-DPP: ; %bb.0:
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s6, 0
+; GFX1132-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
-; GFX1132-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX1132-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s6, vcc_lo, s6
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB9_2
; GFX1132-DPP-NEXT: .LBB9_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -6570,50 +6158,50 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: s_mov_b32 s4, 0
+; GFX7LESS-NEXT: s_brev_b32 s5, 1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX7LESS-NEXT: .LBB10_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s4
-; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s4
-; GFX7LESS-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[4:5]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB10_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s6, s[0:1]
+; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s6
+; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s4
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, s5
+; GFX7LESS-NEXT: v_add_f64 v[3:4], s[2:3], v[3:4]
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s6
+; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v3
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s5, v4
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB10_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB10_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
-; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT: s_mov_b64 s[6:7], 0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[0:3], 0
; GFX7LESS-NEXT: .LBB10_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -s[4:5]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
-; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
+; GFX7LESS-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB10_4
; GFX7LESS-NEXT: .LBB10_5:
; GFX7LESS-NEXT: s_endpgm
@@ -6648,42 +6236,44 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX9-NEXT: .LBB10_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s3, v1, s4
-; GFX9-NEXT: v_readlane_b32 s2, v0, s4
-; GFX9-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: s_mov_b32 s0, 0
+; GFX9-NEXT: s_brev_b32 s1, 1
+; GFX9-NEXT: s_mov_b64 s[2:3], exec
+; GFX9-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9-NEXT: v_readlane_b32 s4, v0, s6
+; GFX9-NEXT: v_readlane_b32 s5, v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_add_f64 v[3:4], s[4:5], v[3:4]
+; GFX9-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v3
+; GFX9-NEXT: v_readfirstlane_b32 s1, v4
; GFX9-NEXT: s_cbranch_scc1 .LBB10_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB10_5
; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX9-NEXT: .LBB10_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB10_4
; GFX9-NEXT: .LBB10_5:
; GFX9-NEXT: s_endpgm
@@ -6718,42 +6308,42 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v4, 0
-; GFX1064-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB10_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX1064-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1064-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_mov_b32 s0, 0
+; GFX1064-NEXT: s_brev_b32 s1, 1
+; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1064-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1064-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1064-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v4
; GFX1064-NEXT: s_cbranch_scc1 .LBB10_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB10_5
; GFX1064-NEXT: ; %bb.3:
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1064-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1064-NEXT: .LBB10_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB10_4
; GFX1064-NEXT: .LBB10_5:
; GFX1064-NEXT: s_endpgm
@@ -6788,41 +6378,41 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v4, 0
-; GFX1032-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB10_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, 0
+; GFX1032-NEXT: s_brev_b32 s1, 1
+; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1032-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1032-NEXT: s_bitset0_b32 s2, s3
+; GFX1032-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v4
; GFX1032-NEXT: s_cbranch_scc1 .LBB10_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB10_5
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
+; GFX1032-NEXT: s_mov_b32 s4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1032-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1032-NEXT: .LBB10_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-NEXT: s_cbranch_execnz .LBB10_4
; GFX1032-NEXT: .LBB10_5:
; GFX1032-NEXT: s_endpgm
@@ -6847,46 +6437,46 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB10_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1164-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_mov_b32 s0, 0
+; GFX1164-NEXT: s_brev_b32 s1, 1
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s6, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1164-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1164-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v4
; GFX1164-NEXT: s_cbranch_scc1 .LBB10_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: ; %bb.2:
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB10_5
; GFX1164-NEXT: ; %bb.3:
-; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1164-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1164-NEXT: .LBB10_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB10_4
; GFX1164-NEXT: .LBB10_5:
; GFX1164-NEXT: s_endpgm
@@ -6912,44 +6502,44 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB10_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s0, 0
+; GFX1132-NEXT: s_brev_b32 s1, 1
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1132-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1132-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1132-NEXT: s_bitset0_b32 s2, s3
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v4
; GFX1132-NEXT: s_cbranch_scc1 .LBB10_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB10_5
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-NEXT: s_mov_b32 s4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1132-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1132-NEXT: .LBB10_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-NEXT: s_cbranch_execnz .LBB10_4
; GFX1132-NEXT: .LBB10_5:
; GFX1132-NEXT: s_endpgm
@@ -7040,70 +6630,72 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1]
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, v3, v1, s[0:1]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v7 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[4:5], v[6:7]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3]
+; GFX9-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX9-DPP-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_add_f64 v[9:10], v[11:12], -s[0:1]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[10:11], -s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v2
; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT: v_mov_b32_e32 v10, v1
; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB10_2
; GFX9-DPP-NEXT: .LBB10_3:
@@ -7140,61 +6732,69 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s[0:1]
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s5, v4, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 32
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], s[2:3], s[4:5]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v7
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: ds_permute_b32 v6, v5, v3
+; GFX1064-DPP-NEXT: ds_permute_b32 v7, v5, v4
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[6:7]
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 63
+; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX1064-DPP-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -v[0:1]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[10:11], -s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, v2
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v1
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB10_2
; GFX1064-DPP-NEXT: .LBB10_3:
; GFX1064-DPP-NEXT: s_endpgm
@@ -7229,56 +6829,58 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s2
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT: v_readlane_b32 s0, v3, 31
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v4, 31
+; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v0, s[2:3]
; GFX1032-DPP-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -v[0:1]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[7:10], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[9:10]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[1:2], v[9:10]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v1
+; GFX1032-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB10_2
; GFX1032-DPP-NEXT: .LBB10_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -7304,72 +6906,86 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s[0:1]
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_permlane64_b32 v5, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, -1, v4
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v4, 32, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v4, 4, v4
+; GFX1164-DPP-NEXT: ds_permute_b32 v5, v4, v2
+; GFX1164-DPP-NEXT: ds_permute_b32 v6, v4, v3
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[5:6]
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT: v_readlane_b32 s3, v3, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b64 v[9:10], v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -v[0:1]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[7:8], v0, v[7:10], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8
; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB10_2
; GFX1164-DPP-NEXT: .LBB10_3:
; GFX1164-DPP-NEXT: s_endpgm
@@ -7395,64 +7011,64 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s2
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s2
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_readlane_b32 s0, v2, 31
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v3, 31
+; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -v[0:1]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v0, v[6:9], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB10_2
; GFX1132-DPP-NEXT: .LBB10_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -7464,48 +7080,41 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp(ptr addrspace(1) %ptr) #1 {
; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT: s_mov_b32 s14, -1
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s7, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB11_3
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
-; GFX7LESS-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
-; GFX7LESS-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, s6
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, s7
; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s8
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s9
; GFX7LESS-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -s[6:7]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB11_2
; GFX7LESS-NEXT: .LBB11_3:
@@ -7513,43 +7122,36 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_one_a
;
; GFX9-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s14, -1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-NEXT: s_add_u32 s12, s12, s11
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s1, v1
; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB11_3
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX9-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NEXT: v_mov_b32_e32 v3, s5
+; GFX9-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
; GFX9-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB11_2
; GFX9-NEXT: .LBB11_3:
; GFX9-NEXT: s_endpgm
@@ -7557,40 +7159,35 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_one_a
; GFX1064-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT: s_mov_b32 s14, -1
-; GFX1064-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB11_3
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
-; GFX1064-NEXT: s_mov_b32 s3, 0x43300000
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3]
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX1064-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v2, s2
-; GFX1064-NEXT: v_mov_b32_e32 v3, s3
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, s4
+; GFX1064-NEXT: v_mov_b32_e32 v3, s5
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB11_2
; GFX1064-NEXT: .LBB11_3:
; GFX1064-NEXT: s_endpgm
@@ -7598,171 +7195,153 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_one_a
; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1032: ; %bb.0:
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT: s_mov_b32 s14, -1
-; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s6, 0
+; GFX1032-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB11_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-NEXT: s_mov_b32 s7, 0x43300000
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v2, s4
; GFX1032-NEXT: v_mov_b32_e32 v3, s5
; GFX1032-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX1032-NEXT: s_cbranch_execnz .LBB11_2
; GFX1032-NEXT: .LBB11_3:
; GFX1032-NEXT: s_endpgm
;
; GFX1164-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1164: ; %bb.0:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_clause 0x1
-; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB11_3
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1164-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1164-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164-NEXT: v_mov_b32_e32 v3, s3
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, s4
+; GFX1164-NEXT: v_mov_b32_e32 v3, s5
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB11_2
; GFX1164-NEXT: .LBB11_3:
; GFX1164-NEXT: s_endpgm
;
; GFX1132-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_clause 0x1
-; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-NEXT: s_mov_b32 s6, 0
+; GFX1132-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v1
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB11_3
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1132-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX1132-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s6, vcc_lo, s6
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
; GFX1132-NEXT: s_cbranch_execnz .LBB11_2
; GFX1132-NEXT: .LBB11_3:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT: s_mov_b32 s14, -1
-; GFX7LESS-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-DPP-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s7, v1
; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
-; GFX7LESS-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, s6
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, s7
; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, s8
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, s9
; GFX7LESS-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[6:7]
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v6
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v4
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v7
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v5
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB11_2
; GFX7LESS-DPP-NEXT: .LBB11_3:
@@ -7770,43 +7349,36 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_one_a
;
; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT: s_mov_b32 s14, -1
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-DPP-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
-; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX9-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s7
; GFX9-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB11_2
; GFX9-DPP-NEXT: .LBB11_3:
; GFX9-DPP-NEXT: s_endpgm
@@ -7814,40 +7386,35 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_one_a
; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1064-DPP: ; %bb.0:
; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1064-DPP-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
-; GFX1064-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX1064-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s2
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s3
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s4
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB11_2
; GFX1064-DPP-NEXT: .LBB11_3:
; GFX1064-DPP-NEXT: s_endpgm
@@ -7855,123 +7422,112 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_one_a
; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1032-DPP: ; %bb.0:
; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s6, 0
+; GFX1032-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s7, 0x43300000
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s4
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s5
; GFX1032-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB11_2
; GFX1032-DPP-NEXT: .LBB11_3:
; GFX1032-DPP-NEXT: s_endpgm
;
; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1164-DPP: ; %bb.0:
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_clause 0x1
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1164-DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s3
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s4
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB11_2
; GFX1164-DPP-NEXT: .LBB11_3:
; GFX1164-DPP-NEXT: s_endpgm
;
; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_clause 0x1
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s6, 0
+; GFX1132-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX1132-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s6, vcc_lo, s6
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB11_2
; GFX1132-DPP-NEXT: .LBB11_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -8010,50 +7566,50 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: s_mov_b32 s4, 0
+; GFX7LESS-NEXT: s_brev_b32 s5, 1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX7LESS-NEXT: .LBB12_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s4
-; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s4
-; GFX7LESS-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[4:5]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB12_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s6, s[0:1]
+; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s6
+; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s4
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, s5
+; GFX7LESS-NEXT: v_add_f64 v[3:4], s[2:3], v[3:4]
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s6
+; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v3
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s5, v4
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB12_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB12_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
-; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT: s_mov_b64 s[6:7], 0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[0:3], 0
; GFX7LESS-NEXT: .LBB12_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -s[4:5]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
-; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
+; GFX7LESS-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB12_4
; GFX7LESS-NEXT: .LBB12_5:
; GFX7LESS-NEXT: s_endpgm
@@ -8088,42 +7644,44 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX9-NEXT: .LBB12_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s3, v1, s4
-; GFX9-NEXT: v_readlane_b32 s2, v0, s4
-; GFX9-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: s_mov_b32 s0, 0
+; GFX9-NEXT: s_brev_b32 s1, 1
+; GFX9-NEXT: s_mov_b64 s[2:3], exec
+; GFX9-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9-NEXT: v_readlane_b32 s4, v0, s6
+; GFX9-NEXT: v_readlane_b32 s5, v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_add_f64 v[3:4], s[4:5], v[3:4]
+; GFX9-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v3
+; GFX9-NEXT: v_readfirstlane_b32 s1, v4
; GFX9-NEXT: s_cbranch_scc1 .LBB12_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB12_5
; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX9-NEXT: .LBB12_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB12_4
; GFX9-NEXT: .LBB12_5:
; GFX9-NEXT: s_endpgm
@@ -8158,42 +7716,42 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v4, 0
-; GFX1064-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB12_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX1064-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1064-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_mov_b32 s0, 0
+; GFX1064-NEXT: s_brev_b32 s1, 1
+; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1064-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1064-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1064-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v4
; GFX1064-NEXT: s_cbranch_scc1 .LBB12_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB12_5
; GFX1064-NEXT: ; %bb.3:
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1064-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1064-NEXT: .LBB12_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB12_4
; GFX1064-NEXT: .LBB12_5:
; GFX1064-NEXT: s_endpgm
@@ -8228,41 +7786,41 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v4, 0
-; GFX1032-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB12_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, 0
+; GFX1032-NEXT: s_brev_b32 s1, 1
+; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1032-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1032-NEXT: s_bitset0_b32 s2, s3
+; GFX1032-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v4
; GFX1032-NEXT: s_cbranch_scc1 .LBB12_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB12_5
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
+; GFX1032-NEXT: s_mov_b32 s4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1032-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1032-NEXT: .LBB12_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-NEXT: s_cbranch_execnz .LBB12_4
; GFX1032-NEXT: .LBB12_5:
; GFX1032-NEXT: s_endpgm
@@ -8287,46 +7845,46 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB12_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1164-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_mov_b32 s0, 0
+; GFX1164-NEXT: s_brev_b32 s1, 1
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s6, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1164-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1164-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v4
; GFX1164-NEXT: s_cbranch_scc1 .LBB12_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: ; %bb.2:
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB12_5
; GFX1164-NEXT: ; %bb.3:
-; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1164-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1164-NEXT: .LBB12_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB12_4
; GFX1164-NEXT: .LBB12_5:
; GFX1164-NEXT: s_endpgm
@@ -8352,44 +7910,44 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB12_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s0, 0
+; GFX1132-NEXT: s_brev_b32 s1, 1
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1132-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1132-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1132-NEXT: s_bitset0_b32 s2, s3
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v4
; GFX1132-NEXT: s_cbranch_scc1 .LBB12_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB12_5
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-NEXT: s_mov_b32 s4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1132-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1132-NEXT: .LBB12_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-NEXT: s_cbranch_execnz .LBB12_4
; GFX1132-NEXT: .LBB12_5:
; GFX1132-NEXT: s_endpgm
@@ -8480,70 +8038,72 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1]
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, v3, v1, s[0:1]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v7 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[4:5], v[6:7]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB12_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3]
+; GFX9-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX9-DPP-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_add_f64 v[9:10], v[11:12], -s[0:1]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[10:11], -s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v2
; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT: v_mov_b32_e32 v10, v1
; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB12_2
; GFX9-DPP-NEXT: .LBB12_3:
@@ -8580,61 +8140,69 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s[0:1]
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s5, v4, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 32
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], s[2:3], s[4:5]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v7
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: ds_permute_b32 v6, v5, v3
+; GFX1064-DPP-NEXT: ds_permute_b32 v7, v5, v4
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[6:7]
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 63
+; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB12_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX1064-DPP-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -v[0:1]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[10:11], -s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, v2
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v1
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB12_2
; GFX1064-DPP-NEXT: .LBB12_3:
; GFX1064-DPP-NEXT: s_endpgm
@@ -8669,56 +8237,58 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s2
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT: v_readlane_b32 s0, v3, 31
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v4, 31
+; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB12_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v0, s[2:3]
; GFX1032-DPP-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -v[0:1]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[7:10], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[9:10]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[1:2], v[9:10]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v1
+; GFX1032-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB12_2
; GFX1032-DPP-NEXT: .LBB12_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -8744,72 +8314,86 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s[0:1]
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_permlane64_b32 v5, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, -1, v4
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v4, 32, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v4, 4, v4
+; GFX1164-DPP-NEXT: ds_permute_b32 v5, v4, v2
+; GFX1164-DPP-NEXT: ds_permute_b32 v6, v4, v3
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[5:6]
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT: v_readlane_b32 s3, v3, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB12_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b64 v[9:10], v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -v[0:1]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[7:8], v0, v[7:10], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8
; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB12_2
; GFX1164-DPP-NEXT: .LBB12_3:
; GFX1164-DPP-NEXT: s_endpgm
@@ -8835,64 +8419,64 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s2
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s2
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_readlane_b32 s0, v2, 31
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v3, 31
+; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB12_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -v[0:1]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v0, v[6:9], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB12_2
; GFX1132-DPP-NEXT: .LBB12_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -8904,48 +8488,41 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp(ptr addrspace(1) %ptr) #2{
; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT: s_mov_b32 s14, -1
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s7, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB13_3
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
-; GFX7LESS-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
-; GFX7LESS-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, s6
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, s7
; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s8
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s9
; GFX7LESS-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -s[6:7]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB13_2
; GFX7LESS-NEXT: .LBB13_3:
@@ -8953,43 +8530,36 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
;
; GFX9-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s14, -1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-NEXT: s_add_u32 s12, s12, s11
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s1, v1
; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB13_3
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX9-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NEXT: v_mov_b32_e32 v3, s5
+; GFX9-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
; GFX9-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB13_2
; GFX9-NEXT: .LBB13_3:
; GFX9-NEXT: s_endpgm
@@ -8997,40 +8567,35 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
; GFX1064-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT: s_mov_b32 s14, -1
-; GFX1064-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB13_3
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
-; GFX1064-NEXT: s_mov_b32 s3, 0x43300000
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3]
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX1064-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v2, s2
-; GFX1064-NEXT: v_mov_b32_e32 v3, s3
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, s4
+; GFX1064-NEXT: v_mov_b32_e32 v3, s5
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB13_2
; GFX1064-NEXT: .LBB13_3:
; GFX1064-NEXT: s_endpgm
@@ -9038,171 +8603,153 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1032: ; %bb.0:
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT: s_mov_b32 s14, -1
-; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s6, 0
+; GFX1032-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB13_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-NEXT: s_mov_b32 s7, 0x43300000
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v2, s4
; GFX1032-NEXT: v_mov_b32_e32 v3, s5
; GFX1032-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX1032-NEXT: s_cbranch_execnz .LBB13_2
; GFX1032-NEXT: .LBB13_3:
; GFX1032-NEXT: s_endpgm
;
; GFX1164-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1164: ; %bb.0:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_clause 0x1
-; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB13_3
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1164-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1164-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164-NEXT: v_mov_b32_e32 v3, s3
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, s4
+; GFX1164-NEXT: v_mov_b32_e32 v3, s5
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB13_2
; GFX1164-NEXT: .LBB13_3:
; GFX1164-NEXT: s_endpgm
;
; GFX1132-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_clause 0x1
-; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-NEXT: s_mov_b32 s6, 0
+; GFX1132-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v1
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB13_3
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1132-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX1132-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s6, vcc_lo, s6
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
; GFX1132-NEXT: s_cbranch_execnz .LBB13_2
; GFX1132-NEXT: .LBB13_3:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT: s_mov_b32 s14, -1
-; GFX7LESS-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-DPP-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s7, v1
; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB13_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
-; GFX7LESS-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, s6
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, s7
; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, s8
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, s9
; GFX7LESS-DPP-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[6:7]
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v6
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v4
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v7
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v5
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB13_2
; GFX7LESS-DPP-NEXT: .LBB13_3:
@@ -9210,43 +8757,36 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
;
; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT: s_mov_b32 s14, -1
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-DPP-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB13_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
-; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX9-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s7
; GFX9-DPP-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB13_2
; GFX9-DPP-NEXT: .LBB13_3:
; GFX9-DPP-NEXT: s_endpgm
@@ -9254,40 +8794,35 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1064-DPP: ; %bb.0:
; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1064-DPP-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB13_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
-; GFX1064-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX1064-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s2
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s3
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s4
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB13_2
; GFX1064-DPP-NEXT: .LBB13_3:
; GFX1064-DPP-NEXT: s_endpgm
@@ -9295,123 +8830,112 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1032-DPP: ; %bb.0:
; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s6, 0
+; GFX1032-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB13_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s7, 0x43300000
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s4
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s5
; GFX1032-DPP-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB13_2
; GFX1032-DPP-NEXT: .LBB13_3:
; GFX1032-DPP-NEXT: s_endpgm
;
; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1164-DPP: ; %bb.0:
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_clause 0x1
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB13_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1164-DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s3
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s4
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB13_2
; GFX1164-DPP-NEXT: .LBB13_3:
; GFX1164-DPP-NEXT: s_endpgm
;
; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_clause 0x1
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s6, 0
+; GFX1132-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB13_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX1132-DPP-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s6, vcc_lo, s6
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB13_2
; GFX1132-DPP-NEXT: .LBB13_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -9451,50 +8975,50 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: s_mov_b32 s4, 0
+; GFX7LESS-NEXT: s_brev_b32 s5, 1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX7LESS-NEXT: .LBB14_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s4
-; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s4
-; GFX7LESS-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[4:5]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB14_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s6, s[0:1]
+; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s6
+; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s4
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, s5
+; GFX7LESS-NEXT: v_add_f64 v[3:4], s[2:3], v[3:4]
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s6
+; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v3
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s5, v4
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB14_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB14_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
-; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT: s_mov_b64 s[6:7], 0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[0:3], 0
; GFX7LESS-NEXT: .LBB14_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -s[4:5]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
-; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
+; GFX7LESS-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB14_4
; GFX7LESS-NEXT: .LBB14_5:
; GFX7LESS-NEXT: s_endpgm
@@ -9529,42 +9053,44 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX9-NEXT: .LBB14_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s3, v1, s4
-; GFX9-NEXT: v_readlane_b32 s2, v0, s4
-; GFX9-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: s_mov_b32 s0, 0
+; GFX9-NEXT: s_brev_b32 s1, 1
+; GFX9-NEXT: s_mov_b64 s[2:3], exec
+; GFX9-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9-NEXT: v_readlane_b32 s4, v0, s6
+; GFX9-NEXT: v_readlane_b32 s5, v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_add_f64 v[3:4], s[4:5], v[3:4]
+; GFX9-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v3
+; GFX9-NEXT: v_readfirstlane_b32 s1, v4
; GFX9-NEXT: s_cbranch_scc1 .LBB14_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB14_5
; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX9-NEXT: .LBB14_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB14_4
; GFX9-NEXT: .LBB14_5:
; GFX9-NEXT: s_endpgm
@@ -9599,42 +9125,42 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v4, 0
-; GFX1064-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB14_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX1064-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1064-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_mov_b32 s0, 0
+; GFX1064-NEXT: s_brev_b32 s1, 1
+; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1064-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1064-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1064-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v4
; GFX1064-NEXT: s_cbranch_scc1 .LBB14_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB14_5
; GFX1064-NEXT: ; %bb.3:
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1064-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1064-NEXT: .LBB14_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB14_4
; GFX1064-NEXT: .LBB14_5:
; GFX1064-NEXT: s_endpgm
@@ -9669,41 +9195,41 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v4, 0
-; GFX1032-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB14_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, 0
+; GFX1032-NEXT: s_brev_b32 s1, 1
+; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1032-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1032-NEXT: s_bitset0_b32 s2, s3
+; GFX1032-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v4
; GFX1032-NEXT: s_cbranch_scc1 .LBB14_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB14_5
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
+; GFX1032-NEXT: s_mov_b32 s4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1032-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1032-NEXT: .LBB14_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-NEXT: s_cbranch_execnz .LBB14_4
; GFX1032-NEXT: .LBB14_5:
; GFX1032-NEXT: s_endpgm
@@ -9728,46 +9254,46 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB14_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1164-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_mov_b32 s0, 0
+; GFX1164-NEXT: s_brev_b32 s1, 1
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s6, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1164-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1164-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v4
; GFX1164-NEXT: s_cbranch_scc1 .LBB14_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: ; %bb.2:
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB14_5
; GFX1164-NEXT: ; %bb.3:
-; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1164-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1164-NEXT: .LBB14_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB14_4
; GFX1164-NEXT: .LBB14_5:
; GFX1164-NEXT: s_endpgm
@@ -9793,44 +9319,44 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB14_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s0, 0
+; GFX1132-NEXT: s_brev_b32 s1, 1
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1132-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1132-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1132-NEXT: s_bitset0_b32 s2, s3
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v4
; GFX1132-NEXT: s_cbranch_scc1 .LBB14_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB14_5
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-NEXT: s_mov_b32 s4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1132-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1132-NEXT: .LBB14_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-NEXT: s_cbranch_execnz .LBB14_4
; GFX1132-NEXT: .LBB14_5:
; GFX1132-NEXT: s_endpgm
@@ -9921,70 +9447,72 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1]
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, v3, v1, s[0:1]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v7 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[4:5], v[6:7]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB14_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3]
+; GFX9-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX9-DPP-NEXT: .LBB14_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_add_f64 v[9:10], v[11:12], -s[0:1]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[10:11], -s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v2
; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT: v_mov_b32_e32 v10, v1
; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB14_2
; GFX9-DPP-NEXT: .LBB14_3:
@@ -10021,61 +9549,69 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s[0:1]
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s5, v4, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 32
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], s[2:3], s[4:5]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v7
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: ds_permute_b32 v6, v5, v3
+; GFX1064-DPP-NEXT: ds_permute_b32 v7, v5, v4
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[6:7]
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 63
+; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB14_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX1064-DPP-NEXT: .LBB14_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -v[0:1]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[10:11], -s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, v2
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v1
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB14_2
; GFX1064-DPP-NEXT: .LBB14_3:
; GFX1064-DPP-NEXT: s_endpgm
@@ -10110,56 +9646,58 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s2
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT: v_readlane_b32 s0, v3, 31
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v4, 31
+; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB14_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v0, s[2:3]
; GFX1032-DPP-NEXT: .LBB14_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -v[0:1]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[7:10], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[9:10]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[1:2], v[9:10]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v1
+; GFX1032-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB14_2
; GFX1032-DPP-NEXT: .LBB14_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -10185,72 +9723,86 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s[0:1]
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_permlane64_b32 v5, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, -1, v4
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v4, 32, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v4, 4, v4
+; GFX1164-DPP-NEXT: ds_permute_b32 v5, v4, v2
+; GFX1164-DPP-NEXT: ds_permute_b32 v6, v4, v3
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[5:6]
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT: v_readlane_b32 s3, v3, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB14_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b64 v[9:10], v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB14_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -v[0:1]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[7:8], v0, v[7:10], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8
; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB14_2
; GFX1164-DPP-NEXT: .LBB14_3:
; GFX1164-DPP-NEXT: s_endpgm
@@ -10276,64 +9828,64 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s2
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s2
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_readlane_b32 s0, v2, 31
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v3, 31
+; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB14_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB14_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -v[0:1]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v0, v[6:9], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB14_2
; GFX1132-DPP-NEXT: .LBB14_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -10374,50 +9926,50 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: s_mov_b32 s4, 0
+; GFX7LESS-NEXT: s_brev_b32 s5, 1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX7LESS-NEXT: .LBB15_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s4
-; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s4
-; GFX7LESS-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[4:5]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB15_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s6, s[0:1]
+; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s6
+; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s4
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, s5
+; GFX7LESS-NEXT: v_add_f64 v[3:4], s[2:3], v[3:4]
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s6
+; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v3
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s5, v4
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB15_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB15_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
-; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT: s_mov_b64 s[6:7], 0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[0:3], 0
; GFX7LESS-NEXT: .LBB15_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -s[4:5]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
-; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
+; GFX7LESS-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB15_4
; GFX7LESS-NEXT: .LBB15_5:
; GFX7LESS-NEXT: s_endpgm
@@ -10452,42 +10004,44 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX9-NEXT: .LBB15_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s3, v1, s4
-; GFX9-NEXT: v_readlane_b32 s2, v0, s4
-; GFX9-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: s_mov_b32 s0, 0
+; GFX9-NEXT: s_brev_b32 s1, 1
+; GFX9-NEXT: s_mov_b64 s[2:3], exec
+; GFX9-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9-NEXT: v_readlane_b32 s4, v0, s6
+; GFX9-NEXT: v_readlane_b32 s5, v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_add_f64 v[3:4], s[4:5], v[3:4]
+; GFX9-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v3
+; GFX9-NEXT: v_readfirstlane_b32 s1, v4
; GFX9-NEXT: s_cbranch_scc1 .LBB15_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB15_5
; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX9-NEXT: .LBB15_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB15_4
; GFX9-NEXT: .LBB15_5:
; GFX9-NEXT: s_endpgm
@@ -10522,42 +10076,42 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v4, 0
-; GFX1064-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB15_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX1064-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1064-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_mov_b32 s0, 0
+; GFX1064-NEXT: s_brev_b32 s1, 1
+; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1064-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1064-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1064-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v4
; GFX1064-NEXT: s_cbranch_scc1 .LBB15_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB15_5
; GFX1064-NEXT: ; %bb.3:
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1064-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1064-NEXT: .LBB15_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB15_4
; GFX1064-NEXT: .LBB15_5:
; GFX1064-NEXT: s_endpgm
@@ -10592,41 +10146,41 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v4, 0
-; GFX1032-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB15_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, 0
+; GFX1032-NEXT: s_brev_b32 s1, 1
+; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1032-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1032-NEXT: s_bitset0_b32 s2, s3
+; GFX1032-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v4
; GFX1032-NEXT: s_cbranch_scc1 .LBB15_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB15_5
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
+; GFX1032-NEXT: s_mov_b32 s4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1032-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1032-NEXT: .LBB15_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-NEXT: s_cbranch_execnz .LBB15_4
; GFX1032-NEXT: .LBB15_5:
; GFX1032-NEXT: s_endpgm
@@ -10651,46 +10205,46 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB15_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1164-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_mov_b32 s0, 0
+; GFX1164-NEXT: s_brev_b32 s1, 1
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s6, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1164-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1164-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v4
; GFX1164-NEXT: s_cbranch_scc1 .LBB15_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: ; %bb.2:
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB15_5
; GFX1164-NEXT: ; %bb.3:
-; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1164-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1164-NEXT: .LBB15_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB15_4
; GFX1164-NEXT: .LBB15_5:
; GFX1164-NEXT: s_endpgm
@@ -10716,44 +10270,44 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB15_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s0, 0
+; GFX1132-NEXT: s_brev_b32 s1, 1
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1132-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1132-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1132-NEXT: s_bitset0_b32 s2, s3
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v4
; GFX1132-NEXT: s_cbranch_scc1 .LBB15_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB15_5
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-NEXT: s_mov_b32 s4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1132-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1132-NEXT: .LBB15_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-NEXT: s_cbranch_execnz .LBB15_4
; GFX1132-NEXT: .LBB15_5:
; GFX1132-NEXT: s_endpgm
@@ -10844,70 +10398,72 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1]
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, v3, v1, s[0:1]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v7 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[4:5], v[6:7]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB15_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3]
+; GFX9-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX9-DPP-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_add_f64 v[9:10], v[11:12], -s[0:1]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[10:11], -s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v2
; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT: v_mov_b32_e32 v10, v1
; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB15_2
; GFX9-DPP-NEXT: .LBB15_3:
@@ -10944,61 +10500,69 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s[0:1]
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s5, v4, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 32
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], s[2:3], s[4:5]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v7
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: ds_permute_b32 v6, v5, v3
+; GFX1064-DPP-NEXT: ds_permute_b32 v7, v5, v4
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[6:7]
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 63
+; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB15_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX1064-DPP-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -v[0:1]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[10:11], -s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, v2
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v1
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB15_2
; GFX1064-DPP-NEXT: .LBB15_3:
; GFX1064-DPP-NEXT: s_endpgm
@@ -11033,56 +10597,58 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s2
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT: v_readlane_b32 s0, v3, 31
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v4, 31
+; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB15_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v0, s[2:3]
; GFX1032-DPP-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -v[0:1]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[7:10], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[9:10]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[1:2], v[9:10]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v1
+; GFX1032-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB15_2
; GFX1032-DPP-NEXT: .LBB15_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -11108,72 +10674,86 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s[0:1]
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_permlane64_b32 v5, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, -1, v4
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v4, 32, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v4, 4, v4
+; GFX1164-DPP-NEXT: ds_permute_b32 v5, v4, v2
+; GFX1164-DPP-NEXT: ds_permute_b32 v6, v4, v3
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[5:6]
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT: v_readlane_b32 s3, v3, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB15_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b64 v[9:10], v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -v[0:1]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[7:8], v0, v[7:10], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8
; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB15_2
; GFX1164-DPP-NEXT: .LBB15_3:
; GFX1164-DPP-NEXT: s_endpgm
@@ -11199,64 +10779,64 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s2
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s2
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_readlane_b32 s0, v2, 31
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v3, 31
+; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB15_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -v[0:1]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v0, v[6:9], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB15_2
; GFX1132-DPP-NEXT: .LBB15_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -11267,48 +10847,41 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp(ptr addrspace(1) %ptr) #2 {
; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT: s_mov_b32 s14, -1
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s7, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB16_3
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
-; GFX7LESS-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
-; GFX7LESS-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, s6
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, s7
; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s8
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s9
; GFX7LESS-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -s[6:7]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB16_2
; GFX7LESS-NEXT: .LBB16_3:
@@ -11316,43 +10889,36 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
;
; GFX9-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s14, -1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-NEXT: s_add_u32 s12, s12, s11
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s1, v1
; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB16_3
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX9-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NEXT: v_mov_b32_e32 v3, s5
+; GFX9-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
; GFX9-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB16_2
; GFX9-NEXT: .LBB16_3:
; GFX9-NEXT: s_endpgm
@@ -11360,40 +10926,35 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
; GFX1064-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT: s_mov_b32 s14, -1
-; GFX1064-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB16_3
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
-; GFX1064-NEXT: s_mov_b32 s3, 0x43300000
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3]
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX1064-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v2, s2
-; GFX1064-NEXT: v_mov_b32_e32 v3, s3
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, s4
+; GFX1064-NEXT: v_mov_b32_e32 v3, s5
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB16_2
; GFX1064-NEXT: .LBB16_3:
; GFX1064-NEXT: s_endpgm
@@ -11401,171 +10962,153 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
; GFX1032: ; %bb.0:
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT: s_mov_b32 s14, -1
-; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s6, 0
+; GFX1032-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB16_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-NEXT: s_mov_b32 s7, 0x43300000
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v2, s4
; GFX1032-NEXT: v_mov_b32_e32 v3, s5
; GFX1032-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX1032-NEXT: s_cbranch_execnz .LBB16_2
; GFX1032-NEXT: .LBB16_3:
; GFX1032-NEXT: s_endpgm
;
; GFX1164-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
; GFX1164: ; %bb.0:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_clause 0x1
-; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB16_3
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1164-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1164-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164-NEXT: v_mov_b32_e32 v3, s3
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, s4
+; GFX1164-NEXT: v_mov_b32_e32 v3, s5
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB16_2
; GFX1164-NEXT: .LBB16_3:
; GFX1164-NEXT: s_endpgm
;
; GFX1132-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_clause 0x1
-; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-NEXT: s_mov_b32 s6, 0
+; GFX1132-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v1
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB16_3
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1132-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX1132-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s6, vcc_lo, s6
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
; GFX1132-NEXT: s_cbranch_execnz .LBB16_2
; GFX1132-NEXT: .LBB16_3:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT: s_mov_b32 s14, -1
-; GFX7LESS-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-DPP-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s7, v1
; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB16_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
-; GFX7LESS-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, s6
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, s7
; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, s8
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, s9
; GFX7LESS-DPP-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[6:7]
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v6
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v4
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v7
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v5
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB16_2
; GFX7LESS-DPP-NEXT: .LBB16_3:
@@ -11573,43 +11116,36 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
;
; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT: s_mov_b32 s14, -1
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-DPP-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB16_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
-; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX9-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s7
; GFX9-DPP-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB16_2
; GFX9-DPP-NEXT: .LBB16_3:
; GFX9-DPP-NEXT: s_endpgm
@@ -11617,40 +11153,35 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
; GFX1064-DPP: ; %bb.0:
; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1064-DPP-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB16_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
-; GFX1064-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX1064-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s2
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s3
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s4
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB16_2
; GFX1064-DPP-NEXT: .LBB16_3:
; GFX1064-DPP-NEXT: s_endpgm
@@ -11658,123 +11189,112 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
; GFX1032-DPP: ; %bb.0:
; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s6, 0
+; GFX1032-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB16_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s7, 0x43300000
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s4
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s5
; GFX1032-DPP-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB16_2
; GFX1032-DPP-NEXT: .LBB16_3:
; GFX1032-DPP-NEXT: s_endpgm
;
; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
; GFX1164-DPP: ; %bb.0:
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_clause 0x1
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB16_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1164-DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s3
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s4
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB16_2
; GFX1164-DPP-NEXT: .LBB16_3:
; GFX1164-DPP-NEXT: s_endpgm
;
; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_clause 0x1
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s6, 0
+; GFX1132-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB16_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX1132-DPP-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s6, vcc_lo, s6
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB16_2
; GFX1132-DPP-NEXT: .LBB16_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -11814,50 +11334,50 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: s_mov_b32 s4, 0
+; GFX7LESS-NEXT: s_brev_b32 s5, 1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX7LESS-NEXT: .LBB17_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s4
-; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s4
-; GFX7LESS-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[4:5]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB17_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s6, s[0:1]
+; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s6
+; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s4
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, s5
+; GFX7LESS-NEXT: v_add_f64 v[3:4], s[2:3], v[3:4]
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s6
+; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v3
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s5, v4
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB17_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB17_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
-; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT: s_mov_b64 s[6:7], 0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[0:3], 0
; GFX7LESS-NEXT: .LBB17_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -s[4:5]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
-; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
+; GFX7LESS-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB17_4
; GFX7LESS-NEXT: .LBB17_5:
; GFX7LESS-NEXT: s_endpgm
@@ -11892,42 +11412,44 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX9-NEXT: .LBB17_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s3, v1, s4
-; GFX9-NEXT: v_readlane_b32 s2, v0, s4
-; GFX9-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: s_mov_b32 s0, 0
+; GFX9-NEXT: s_brev_b32 s1, 1
+; GFX9-NEXT: s_mov_b64 s[2:3], exec
+; GFX9-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9-NEXT: v_readlane_b32 s4, v0, s6
+; GFX9-NEXT: v_readlane_b32 s5, v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_add_f64 v[3:4], s[4:5], v[3:4]
+; GFX9-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v3
+; GFX9-NEXT: v_readfirstlane_b32 s1, v4
; GFX9-NEXT: s_cbranch_scc1 .LBB17_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB17_5
; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX9-NEXT: .LBB17_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB17_4
; GFX9-NEXT: .LBB17_5:
; GFX9-NEXT: s_endpgm
@@ -11962,42 +11484,42 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v4, 0
-; GFX1064-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB17_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX1064-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1064-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_mov_b32 s0, 0
+; GFX1064-NEXT: s_brev_b32 s1, 1
+; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1064-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1064-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1064-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v4
; GFX1064-NEXT: s_cbranch_scc1 .LBB17_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB17_5
; GFX1064-NEXT: ; %bb.3:
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1064-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1064-NEXT: .LBB17_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB17_4
; GFX1064-NEXT: .LBB17_5:
; GFX1064-NEXT: s_endpgm
@@ -12032,41 +11554,41 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v4, 0
-; GFX1032-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB17_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, 0
+; GFX1032-NEXT: s_brev_b32 s1, 1
+; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1032-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1032-NEXT: s_bitset0_b32 s2, s3
+; GFX1032-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v4
; GFX1032-NEXT: s_cbranch_scc1 .LBB17_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB17_5
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
+; GFX1032-NEXT: s_mov_b32 s4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1032-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1032-NEXT: .LBB17_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-NEXT: s_cbranch_execnz .LBB17_4
; GFX1032-NEXT: .LBB17_5:
; GFX1032-NEXT: s_endpgm
@@ -12091,46 +11613,46 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB17_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1164-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_mov_b32 s0, 0
+; GFX1164-NEXT: s_brev_b32 s1, 1
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s6, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1164-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1164-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v4
; GFX1164-NEXT: s_cbranch_scc1 .LBB17_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: ; %bb.2:
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB17_5
; GFX1164-NEXT: ; %bb.3:
-; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1164-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1164-NEXT: .LBB17_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB17_4
; GFX1164-NEXT: .LBB17_5:
; GFX1164-NEXT: s_endpgm
@@ -12156,44 +11678,44 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB17_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s0, 0
+; GFX1132-NEXT: s_brev_b32 s1, 1
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1132-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1132-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1132-NEXT: s_bitset0_b32 s2, s3
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v4
; GFX1132-NEXT: s_cbranch_scc1 .LBB17_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB17_5
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-NEXT: s_mov_b32 s4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1132-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1132-NEXT: .LBB17_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-NEXT: s_cbranch_execnz .LBB17_4
; GFX1132-NEXT: .LBB17_5:
; GFX1132-NEXT: s_endpgm
@@ -12284,70 +11806,72 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1]
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, v3, v1, s[0:1]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v7 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[4:5], v[6:7]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB17_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3]
+; GFX9-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX9-DPP-NEXT: .LBB17_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_add_f64 v[9:10], v[11:12], -s[0:1]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[10:11], -s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v2
; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT: v_mov_b32_e32 v10, v1
; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB17_2
; GFX9-DPP-NEXT: .LBB17_3:
@@ -12384,61 +11908,69 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s[0:1]
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s5, v4, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 32
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], s[2:3], s[4:5]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v7
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: ds_permute_b32 v6, v5, v3
+; GFX1064-DPP-NEXT: ds_permute_b32 v7, v5, v4
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[6:7]
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 63
+; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB17_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX1064-DPP-NEXT: .LBB17_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -v[0:1]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[10:11], -s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, v2
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v1
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB17_2
; GFX1064-DPP-NEXT: .LBB17_3:
; GFX1064-DPP-NEXT: s_endpgm
@@ -12473,56 +12005,58 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s2
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT: v_readlane_b32 s0, v3, 31
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v4, 31
+; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB17_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v0, s[2:3]
; GFX1032-DPP-NEXT: .LBB17_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -v[0:1]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[7:10], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[9:10]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[1:2], v[9:10]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v1
+; GFX1032-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB17_2
; GFX1032-DPP-NEXT: .LBB17_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -12548,72 +12082,86 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s[0:1]
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_permlane64_b32 v5, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, -1, v4
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v4, 32, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v4, 4, v4
+; GFX1164-DPP-NEXT: ds_permute_b32 v5, v4, v2
+; GFX1164-DPP-NEXT: ds_permute_b32 v6, v4, v3
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[5:6]
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT: v_readlane_b32 s3, v3, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB17_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b64 v[9:10], v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB17_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -v[0:1]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[7:8], v0, v[7:10], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8
; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB17_2
; GFX1164-DPP-NEXT: .LBB17_3:
; GFX1164-DPP-NEXT: s_endpgm
@@ -12639,64 +12187,64 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s2
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s2
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_readlane_b32 s0, v2, 31
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v3, 31
+; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB17_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB17_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -v[0:1]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v0, v[6:9], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB17_2
; GFX1132-DPP-NEXT: .LBB17_3:
; GFX1132-DPP-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_precise_memory.ll b/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_precise_memory.ll
index faa653d11b52a..488013538ea8f 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_precise_memory.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_precise_memory.ll
@@ -682,19 +682,19 @@ main_body:
define amdgpu_kernel void @atomic_add_local(ptr addrspace(3) %local) {
; GFX9-LABEL: atomic_add_local:
; GFX9: ; %bb.0:
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB5_2
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s1, s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
; GFX9-NEXT: s_mul_i32 s0, s0, 5
; GFX9-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
; GFX9-NEXT: ds_add_u32 v0, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: .LBB5_2:
@@ -702,19 +702,19 @@ define amdgpu_kernel void @atomic_add_local(ptr addrspace(3) %local) {
;
; GFX90A-LABEL: atomic_add_local:
; GFX90A: ; %bb.0:
+; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
-; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB5_2
; GFX90A-NEXT: ; %bb.1:
-; GFX90A-NEXT: s_load_dword s2, s[4:5], 0x24
+; GFX90A-NEXT: s_load_dword s1, s[4:5], 0x24
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
; GFX90A-NEXT: s_mul_i32 s0, s0, 5
; GFX90A-NEXT: v_mov_b32_e32 v1, s0
-; GFX90A-NEXT: v_mov_b32_e32 v0, s2
+; GFX90A-NEXT: v_mov_b32_e32 v0, s1
; GFX90A-NEXT: ds_add_u32 v0, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: .LBB5_2:
@@ -722,15 +722,15 @@ define amdgpu_kernel void @atomic_add_local(ptr addrspace(3) %local) {
;
; GFX10-LABEL: atomic_add_local:
; GFX10: ; %bb.0:
+; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX10-NEXT: s_mov_b32 s0, exec_lo
-; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX10-NEXT: s_bcnt1_i32_b32 s0, s0
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB5_2
; GFX10-NEXT: ; %bb.1:
; GFX10-NEXT: s_load_dword s1, s[4:5], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_bcnt1_i32_b32 s0, s0
; GFX10-NEXT: s_mul_i32 s0, s0, 5
; GFX10-NEXT: v_mov_b32_e32 v1, s0
; GFX10-NEXT: v_mov_b32_e32 v0, s1
@@ -742,19 +742,19 @@ define amdgpu_kernel void @atomic_add_local(ptr addrspace(3) %local) {
;
; GFX9-FLATSCR-LABEL: atomic_add_local:
; GFX9-FLATSCR: ; %bb.0:
+; GFX9-FLATSCR-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-FLATSCR-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-FLATSCR-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-FLATSCR-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-FLATSCR-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX9-FLATSCR-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
; GFX9-FLATSCR-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-FLATSCR-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-FLATSCR-NEXT: s_cbranch_execz .LBB5_2
; GFX9-FLATSCR-NEXT: ; %bb.1:
-; GFX9-FLATSCR-NEXT: s_load_dword s2, s[4:5], 0x24
+; GFX9-FLATSCR-NEXT: s_load_dword s1, s[4:5], 0x24
; GFX9-FLATSCR-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-FLATSCR-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
; GFX9-FLATSCR-NEXT: s_mul_i32 s0, s0, 5
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v0, s1
; GFX9-FLATSCR-NEXT: ds_add_u32 v0, v1
; GFX9-FLATSCR-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-FLATSCR-NEXT: .LBB5_2:
@@ -762,18 +762,18 @@ define amdgpu_kernel void @atomic_add_local(ptr addrspace(3) %local) {
;
; GFX11-LABEL: atomic_add_local:
; GFX11: ; %bb.0:
+; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11-NEXT: s_mov_b32 s0, exec_lo
; GFX11-NEXT: s_mov_b32 s1, exec_lo
-; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX11-NEXT: s_bcnt1_i32_b32 s0, s0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11-NEXT: s_cbranch_execz .LBB5_2
; GFX11-NEXT: ; %bb.1:
; GFX11-NEXT: s_load_b32 s1, s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_mul_i32 s0, s0, 5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v0, s1
; GFX11-NEXT: ds_add_u32 v0, v1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
@@ -783,19 +783,18 @@ define amdgpu_kernel void @atomic_add_local(ptr addrspace(3) %local) {
;
; GFX12-LABEL: atomic_add_local:
; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12-NEXT: s_mov_b32 s0, exec_lo
; GFX12-NEXT: s_mov_b32 s1, exec_lo
-; GFX12-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX12-NEXT: s_bcnt1_i32_b32 s0, s0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12-NEXT: s_cbranch_execz .LBB5_2
; GFX12-NEXT: ; %bb.1:
; GFX12-NEXT: s_load_b32 s1, s[4:5], 0x24
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_mul_i32 s0, s0, 5
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v0, s1
; GFX12-NEXT: ds_add_u32 v0, v1
; GFX12-NEXT: s_wait_dscnt 0x0
@@ -882,20 +881,20 @@ define void @flat_atomic_xchg_i32_noret(ptr %ptr, i32 %in) {
define amdgpu_kernel void @atomic_add_ret_local(ptr addrspace(1) %out, ptr addrspace(3) %local) {
; GFX9-LABEL: atomic_add_ret_local:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB7_2
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dword s3, s[4:5], 0x2c
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_mul_i32 s2, s2, 5
; GFX9-NEXT: v_mov_b32_e32 v2, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
; GFX9-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: .LBB7_2:
@@ -911,20 +910,20 @@ define amdgpu_kernel void @atomic_add_ret_local(ptr addrspace(1) %out, ptr addrs
;
; GFX90A-LABEL: atomic_add_ret_local:
; GFX90A: ; %bb.0:
-; GFX90A-NEXT: s_mov_b64 s[2:3], exec
-; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX90A-NEXT: s_mov_b64 s[0:1], exec
+; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT: ; implicit-def: $vgpr1
; GFX90A-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB7_2
; GFX90A-NEXT: ; %bb.1:
-; GFX90A-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX90A-NEXT: s_load_dword s3, s[4:5], 0x2c
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX90A-NEXT: s_mul_i32 s2, s2, 5
; GFX90A-NEXT: v_mov_b32_e32 v2, s2
-; GFX90A-NEXT: v_mov_b32_e32 v1, s6
+; GFX90A-NEXT: v_mov_b32_e32 v1, s3
; GFX90A-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: .LBB7_2:
@@ -940,16 +939,16 @@ define amdgpu_kernel void @atomic_add_ret_local(ptr addrspace(1) %out, ptr addrs
;
; GFX10-LABEL: atomic_add_ret_local:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_mov_b32 s1, exec_lo
+; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10-NEXT: s_mov_b32 s0, exec_lo
; GFX10-NEXT: ; implicit-def: $vgpr1
-; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
+; GFX10-NEXT: s_bcnt1_i32_b32 s1, s0
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB7_2
; GFX10-NEXT: ; %bb.1:
; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_bcnt1_i32_b32 s1, s1
; GFX10-NEXT: s_mul_i32 s1, s1, 5
; GFX10-NEXT: v_mov_b32_e32 v2, s1
; GFX10-NEXT: v_mov_b32_e32 v1, s2
@@ -970,20 +969,20 @@ define amdgpu_kernel void @atomic_add_ret_local(ptr addrspace(1) %out, ptr addrs
;
; GFX9-FLATSCR-LABEL: atomic_add_ret_local:
; GFX9-FLATSCR: ; %bb.0:
-; GFX9-FLATSCR-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-FLATSCR-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-FLATSCR-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-FLATSCR-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-FLATSCR-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-FLATSCR-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-FLATSCR-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX9-FLATSCR-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-FLATSCR-NEXT: ; implicit-def: $vgpr1
; GFX9-FLATSCR-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-FLATSCR-NEXT: s_cbranch_execz .LBB7_2
; GFX9-FLATSCR-NEXT: ; %bb.1:
-; GFX9-FLATSCR-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-FLATSCR-NEXT: s_load_dword s3, s[4:5], 0x2c
; GFX9-FLATSCR-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-FLATSCR-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-FLATSCR-NEXT: s_mul_i32 s2, s2, 5
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v2, s2
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v1, s3
; GFX9-FLATSCR-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX9-FLATSCR-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-FLATSCR-NEXT: .LBB7_2:
@@ -999,19 +998,19 @@ define amdgpu_kernel void @atomic_add_ret_local(ptr addrspace(1) %out, ptr addrs
;
; GFX11-LABEL: atomic_add_ret_local:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_mov_b32 s1, exec_lo
+; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX11-NEXT: ; implicit-def: $vgpr1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX11-NEXT: s_mov_b32 s0, exec_lo
; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11-NEXT: s_cbranch_execz .LBB7_2
; GFX11-NEXT: ; %bb.1:
; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_mul_i32 s1, s1, 5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
@@ -1030,20 +1029,19 @@ define amdgpu_kernel void @atomic_add_ret_local(ptr addrspace(1) %out, ptr addrs
;
; GFX12-LABEL: atomic_add_ret_local:
; GFX12: ; %bb.0:
-; GFX12-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12-NEXT: s_mov_b32 s0, exec_lo
-; GFX12-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX12-NEXT: ; implicit-def: $vgpr1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX12-NEXT: s_mov_b32 s0, exec_lo
; GFX12-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12-NEXT: s_cbranch_execz .LBB7_2
; GFX12-NEXT: ; %bb.1:
; GFX12-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_mul_i32 s1, s1, 5
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s2
; GFX12-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX12-NEXT: s_wait_dscnt 0x0
@@ -1074,9 +1072,10 @@ declare i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add(i32, ptr addrspace(8), i32, i
define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace(8) %inout) {
; GFX9-LABEL: add_i32_constant:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
@@ -1084,7 +1083,6 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_mul_i32 s2, s2, 5
; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
@@ -1102,9 +1100,10 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX90A-LABEL: add_i32_constant:
; GFX90A: ; %bb.0: ; %entry
-; GFX90A-NEXT: s_mov_b64 s[2:3], exec
-; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX90A-NEXT: s_mov_b64 s[0:1], exec
+; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT: ; implicit-def: $vgpr1
; GFX90A-NEXT: s_and_saveexec_b64 s[0:1], vcc
@@ -1112,7 +1111,6 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX90A-NEXT: s_mul_i32 s2, s2, 5
; GFX90A-NEXT: v_mov_b32_e32 v1, s2
; GFX90A-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
@@ -1130,16 +1128,16 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX10-LABEL: add_i32_constant:
; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_mov_b32 s1, exec_lo
+; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10-NEXT: s_mov_b32 s0, exec_lo
; GFX10-NEXT: ; implicit-def: $vgpr1
-; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
+; GFX10-NEXT: s_bcnt1_i32_b32 s1, s0
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB8_2
; GFX10-NEXT: ; %bb.1:
; GFX10-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_bcnt1_i32_b32 s1, s1
; GFX10-NEXT: s_mul_i32 s1, s1, 5
; GFX10-NEXT: v_mov_b32_e32 v1, s1
; GFX10-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
@@ -1158,9 +1156,10 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX9-FLATSCR-LABEL: add_i32_constant:
; GFX9-FLATSCR: ; %bb.0: ; %entry
-; GFX9-FLATSCR-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-FLATSCR-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-FLATSCR-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-FLATSCR-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-FLATSCR-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-FLATSCR-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-FLATSCR-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX9-FLATSCR-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-FLATSCR-NEXT: ; implicit-def: $vgpr1
; GFX9-FLATSCR-NEXT: s_and_saveexec_b64 s[0:1], vcc
@@ -1168,7 +1167,6 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX9-FLATSCR-NEXT: ; %bb.1:
; GFX9-FLATSCR-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
; GFX9-FLATSCR-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-FLATSCR-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-FLATSCR-NEXT: s_mul_i32 s2, s2, 5
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v1, s2
; GFX9-FLATSCR-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
@@ -1186,19 +1184,19 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX11-LABEL: add_i32_constant:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_mov_b32 s1, exec_lo
+; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX11-NEXT: ; implicit-def: $vgpr1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX11-NEXT: s_mov_b32 s0, exec_lo
; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11-NEXT: s_cbranch_execz .LBB8_2
; GFX11-NEXT: ; %bb.1:
; GFX11-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_mul_i32 s1, s1, 5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_mov_b32_e32 v1, s1
; GFX11-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], 0 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -1216,20 +1214,19 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX12-LABEL: add_i32_constant:
; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12-NEXT: s_mov_b32 s0, exec_lo
-; GFX12-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX12-NEXT: ; implicit-def: $vgpr1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX12-NEXT: s_mov_b32 s0, exec_lo
; GFX12-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12-NEXT: s_cbranch_execz .LBB8_2
; GFX12-NEXT: ; %bb.1:
; GFX12-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_mul_i32 s1, s1, 5
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: v_mov_b32_e32 v1, s1
; GFX12-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
index 754eadf60a309..365f7b3b7141d 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
@@ -8166,44 +8166,51 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX12-LABEL: local_ds_fadd:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
-; GFX12-NEXT: s_mov_b32 s6, exec_lo
-; GFX12-NEXT: ; implicit-def: $vgpr1
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX12-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX12-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_mul_f32_e32 v1, 0x42280000, v0
+; GFX12-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX12-NEXT: v_readfirstlane_b32 s0, v1
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: ; implicit-def: $vgpr1
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_add_co_i32 s1, s3, 4
-; GFX12-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX12-NEXT: s_and_saveexec_b32 s3, vcc_lo
; GFX12-NEXT: s_cbranch_execz .LBB28_2
; GFX12-NEXT: ; %bb.1:
-; GFX12-NEXT: s_bcnt1_i32_b32 s3, s6
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cvt_f32_ubyte0_e32 v1, s3
-; GFX12-NEXT: s_lshl_b32 s3, s1, 3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mul_f32 v1, 0x42280000, v1
-; GFX12-NEXT: ds_add_rtn_f32 v1, v2, v1
+; GFX12-NEXT: s_lshl_b32 s6, s1, 3
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v1, s6
+; GFX12-NEXT: ds_add_rtn_f32 v1, v1, v2
; GFX12-NEXT: s_wait_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: .LBB28_2:
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: s_mov_b32 s7, exec_lo
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-NEXT: v_mbcnt_lo_u32_b32 v3, exec_lo, 0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_bcnt1_i32_b32 s0, s0
; GFX12-NEXT: v_readfirstlane_b32 s3, v1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mbcnt_lo_u32_b32 v2, s7, 0
+; GFX12-NEXT: v_cvt_f32_i32_e32 v2, s0
; GFX12-NEXT: s_mov_b32 s6, exec_lo
-; GFX12-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX12-NEXT: v_readfirstlane_b32 s7, v2
+; GFX12-NEXT: v_cmpx_eq_u32_e32 0, v3
; GFX12-NEXT: s_cbranch_execz .LBB28_4
; GFX12-NEXT: ; %bb.3:
-; GFX12-NEXT: s_bcnt1_i32_b32 s0, s7
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX12-NEXT: s_lshl_b32 s0, s1, 4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mul_f32 v1, 0x42280000, v1
+; GFX12-NEXT: v_dual_mov_b32 v2, s7 :: v_dual_mov_b32 v1, s0
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ds_add_f32 v2, v1
+; GFX12-NEXT: ds_add_f32 v1, v2
; GFX12-NEXT: s_wait_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: .LBB28_4:
@@ -8261,47 +8268,51 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
;
; GFX942-LABEL: local_ds_fadd:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
; GFX942-NEXT: s_mov_b64 s[0:1], exec
-; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX942-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX942-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX942-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_add_i32 s3, s3, 4
+; GFX942-NEXT: v_readfirstlane_b32 s6, v1
; GFX942-NEXT: ; implicit-def: $vgpr1
-; GFX942-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX942-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX942-NEXT: s_cbranch_execz .LBB28_2
; GFX942-NEXT: ; %bb.1:
-; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX942-NEXT: s_lshl_b32 s8, s3, 3
-; GFX942-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
-; GFX942-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX942-NEXT: v_mov_b32_e32 v2, s8
-; GFX942-NEXT: ds_add_rtn_f32 v1, v2, v1
+; GFX942-NEXT: s_lshl_b32 s7, s3, 3
+; GFX942-NEXT: v_mov_b32_e32 v1, s7
+; GFX942-NEXT: v_mov_b32_e32 v2, s6
+; GFX942-NEXT: ds_add_rtn_f32 v1, v1, v2
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: .LBB28_2:
-; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX942-NEXT: s_mov_b64 s[8:9], exec
-; GFX942-NEXT: v_readfirstlane_b32 s10, v1
-; GFX942-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 0
-; GFX942-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v1
+; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_mov_b64 s[0:1], exec
+; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX942-NEXT: v_cvt_f32_i32_e32 v2, s0
+; GFX942-NEXT: v_readfirstlane_b32 s8, v1
+; GFX942-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX942-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX942-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
; GFX942-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1
+; GFX942-NEXT: v_readfirstlane_b32 s9, v2
; GFX942-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
; GFX942-NEXT: s_cbranch_execz .LBB28_4
; GFX942-NEXT: ; %bb.3:
-; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
-; GFX942-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX942-NEXT: s_lshl_b32 s0, s3, 4
-; GFX942-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX942-NEXT: v_mov_b32_e32 v2, s0
-; GFX942-NEXT: ds_add_f32 v2, v1
+; GFX942-NEXT: v_mov_b32_e32 v1, s0
+; GFX942-NEXT: v_mov_b32_e32 v2, s9
+; GFX942-NEXT: ds_add_f32 v1, v2
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: .LBB28_4:
; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX942-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX942-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
-; GFX942-NEXT: v_add_f32_e32 v0, s10, v0
-; GFX942-NEXT: v_mov_b32_e32 v1, s10
+; GFX942-NEXT: v_add_f32_e32 v0, s8, v0
+; GFX942-NEXT: v_mov_b32_e32 v1, s8
; GFX942-NEXT: s_mov_b64 s[0:1], exec
; GFX942-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc
; GFX942-NEXT: v_bfrev_b32_e32 v1, 1
@@ -8345,42 +8356,47 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX11-LABEL: local_ds_fadd:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
-; GFX11-NEXT: s_mov_b32 s6, exec_lo
-; GFX11-NEXT: ; implicit-def: $vgpr1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX11-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX11-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_mul_f32_e32 v1, 0x42280000, v0
+; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX11-NEXT: v_readfirstlane_b32 s0, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX11-NEXT: ; implicit-def: $vgpr1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_add_i32 s1, s3, 4
-; GFX11-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT: s_and_saveexec_b32 s3, vcc_lo
; GFX11-NEXT: s_cbranch_execz .LBB28_2
; GFX11-NEXT: ; %bb.1:
-; GFX11-NEXT: s_bcnt1_i32_b32 s3, s6
+; GFX11-NEXT: s_lshl_b32 s6, s1, 3
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s3
-; GFX11-NEXT: s_lshl_b32 s3, s1, 3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mul_f32 v1, 0x42280000, v1
-; GFX11-NEXT: ds_add_rtn_f32 v1, v2, v1
+; GFX11-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v1, s6
+; GFX11-NEXT: ds_add_rtn_f32 v1, v1, v2
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: .LBB28_2:
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: s_mov_b32 s7, exec_lo
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-NEXT: v_mbcnt_lo_u32_b32 v3, exec_lo, 0
+; GFX11-NEXT: s_bcnt1_i32_b32 s0, s0
; GFX11-NEXT: v_readfirstlane_b32 s3, v1
-; GFX11-NEXT: v_mbcnt_lo_u32_b32 v2, s7, 0
+; GFX11-NEXT: v_cvt_f32_i32_e32 v2, s0
; GFX11-NEXT: s_mov_b32 s6, exec_lo
-; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX11-NEXT: v_readfirstlane_b32 s7, v2
+; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v3
; GFX11-NEXT: s_cbranch_execz .LBB28_4
; GFX11-NEXT: ; %bb.3:
-; GFX11-NEXT: s_bcnt1_i32_b32 s0, s7
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX11-NEXT: s_lshl_b32 s0, s1, 4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mul_f32 v1, 0x42280000, v1
-; GFX11-NEXT: ds_add_f32 v2, v1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_dual_mov_b32 v2, s7 :: v_dual_mov_b32 v1, s0
+; GFX11-NEXT: ds_add_f32 v1, v2
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: .LBB28_4:
@@ -8433,40 +8449,44 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX10-LABEL: local_ds_fadd:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
-; GFX10-NEXT: ; implicit-def: $vgpr1
-; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX10-NEXT: s_mov_b32 s0, exec_lo
+; GFX10-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX10-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX10-NEXT: v_mul_f32_e32 v1, 0x42280000, v0
+; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10-NEXT: v_readfirstlane_b32 s0, v1
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-NEXT: ; implicit-def: $vgpr1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_add_i32 s1, s3, 4
-; GFX10-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX10-NEXT: s_and_saveexec_b32 s3, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB28_2
; GFX10-NEXT: ; %bb.1:
-; GFX10-NEXT: s_bcnt1_i32_b32 s3, s6
-; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s3
-; GFX10-NEXT: s_lshl_b32 s3, s1, 3
-; GFX10-NEXT: v_mov_b32_e32 v2, s3
-; GFX10-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX10-NEXT: ds_add_rtn_f32 v1, v2, v1
+; GFX10-NEXT: s_lshl_b32 s6, s1, 3
+; GFX10-NEXT: v_mov_b32_e32 v2, s0
+; GFX10-NEXT: v_mov_b32_e32 v1, s6
+; GFX10-NEXT: ds_add_rtn_f32 v1, v1, v2
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: .LBB28_2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX10-NEXT: s_mov_b32 s7, exec_lo
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX10-NEXT: s_mov_b32 s0, exec_lo
+; GFX10-NEXT: v_mbcnt_lo_u32_b32 v3, exec_lo, 0
+; GFX10-NEXT: s_bcnt1_i32_b32 s0, s0
; GFX10-NEXT: v_readfirstlane_b32 s3, v1
-; GFX10-NEXT: v_mbcnt_lo_u32_b32 v2, s7, 0
-; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v2
+; GFX10-NEXT: v_cvt_f32_i32_e32 v2, s0
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v3
+; GFX10-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v2
; GFX10-NEXT: s_and_saveexec_b32 s6, s0
; GFX10-NEXT: s_cbranch_execz .LBB28_4
; GFX10-NEXT: ; %bb.3:
-; GFX10-NEXT: s_bcnt1_i32_b32 s0, s7
-; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX10-NEXT: s_lshl_b32 s0, s1, 4
-; GFX10-NEXT: v_mov_b32_e32 v2, s0
-; GFX10-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
+; GFX10-NEXT: v_mov_b32_e32 v2, s7
+; GFX10-NEXT: v_mov_b32_e32 v1, s0
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: ds_add_f32 v2, v1
+; GFX10-NEXT: ds_add_f32 v1, v2
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: .LBB28_4:
@@ -8516,47 +8536,51 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
;
; GFX90A-LABEL: local_ds_fadd:
; GFX90A: ; %bb.0:
-; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
-; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX90A-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX90A-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: s_add_i32 s3, s3, 4
+; GFX90A-NEXT: v_readfirstlane_b32 s6, v1
; GFX90A-NEXT: ; implicit-def: $vgpr1
-; GFX90A-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX90A-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB28_2
; GFX90A-NEXT: ; %bb.1:
-; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX90A-NEXT: s_lshl_b32 s8, s3, 3
-; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
-; GFX90A-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX90A-NEXT: v_mov_b32_e32 v2, s8
-; GFX90A-NEXT: ds_add_rtn_f32 v1, v2, v1
+; GFX90A-NEXT: s_lshl_b32 s7, s3, 3
+; GFX90A-NEXT: v_mov_b32_e32 v1, s7
+; GFX90A-NEXT: v_mov_b32_e32 v2, s6
+; GFX90A-NEXT: ds_add_rtn_f32 v1, v1, v2
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: .LBB28_2:
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_mov_b64 s[8:9], exec
-; GFX90A-NEXT: v_readfirstlane_b32 s10, v1
-; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 0
-; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v1
+; GFX90A-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX90A-NEXT: s_mov_b64 s[0:1], exec
+; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX90A-NEXT: v_cvt_f32_i32_e32 v2, s0
+; GFX90A-NEXT: v_readfirstlane_b32 s8, v1
+; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX90A-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX90A-NEXT: v_readfirstlane_b32 s9, v2
; GFX90A-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1
; GFX90A-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
; GFX90A-NEXT: s_cbranch_execz .LBB28_4
; GFX90A-NEXT: ; %bb.3:
-; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
-; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX90A-NEXT: s_lshl_b32 s0, s3, 4
-; GFX90A-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX90A-NEXT: v_mov_b32_e32 v2, s0
-; GFX90A-NEXT: ds_add_f32 v2, v1
+; GFX90A-NEXT: v_mov_b32_e32 v1, s0
+; GFX90A-NEXT: v_mov_b32_e32 v2, s9
+; GFX90A-NEXT: ds_add_f32 v1, v2
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: .LBB28_4:
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX90A-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
-; GFX90A-NEXT: v_add_f32_e32 v0, s10, v0
-; GFX90A-NEXT: v_mov_b32_e32 v1, s10
+; GFX90A-NEXT: v_add_f32_e32 v0, s8, v0
+; GFX90A-NEXT: v_mov_b32_e32 v1, s8
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
; GFX90A-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc
; GFX90A-NEXT: v_bfrev_b32_e32 v1, 1
@@ -8598,47 +8622,51 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
;
; GFX908-LABEL: local_ds_fadd:
; GFX908: ; %bb.0:
-; GFX908-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
; GFX908-NEXT: s_mov_b64 s[0:1], exec
-; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX908-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX908-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX908-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX908-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_add_i32 s3, s3, 4
+; GFX908-NEXT: v_readfirstlane_b32 s6, v1
; GFX908-NEXT: ; implicit-def: $vgpr1
-; GFX908-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX908-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX908-NEXT: s_cbranch_execz .LBB28_2
; GFX908-NEXT: ; %bb.1:
-; GFX908-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX908-NEXT: s_lshl_b32 s8, s3, 3
-; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
-; GFX908-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX908-NEXT: v_mov_b32_e32 v2, s8
-; GFX908-NEXT: ds_add_rtn_f32 v1, v2, v1
+; GFX908-NEXT: s_lshl_b32 s7, s3, 3
+; GFX908-NEXT: v_mov_b32_e32 v1, s7
+; GFX908-NEXT: v_mov_b32_e32 v2, s6
+; GFX908-NEXT: ds_add_rtn_f32 v1, v1, v2
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: .LBB28_2:
-; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_mov_b64 s[8:9], exec
-; GFX908-NEXT: v_readfirstlane_b32 s10, v1
-; GFX908-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 0
-; GFX908-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v1
+; GFX908-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX908-NEXT: s_mov_b64 s[0:1], exec
+; GFX908-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX908-NEXT: v_cvt_f32_i32_e32 v2, s0
+; GFX908-NEXT: v_readfirstlane_b32 s8, v1
+; GFX908-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX908-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX908-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX908-NEXT: v_readfirstlane_b32 s9, v2
; GFX908-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1
; GFX908-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
; GFX908-NEXT: s_cbranch_execz .LBB28_4
; GFX908-NEXT: ; %bb.3:
-; GFX908-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
-; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX908-NEXT: s_lshl_b32 s0, s3, 4
-; GFX908-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX908-NEXT: v_mov_b32_e32 v2, s0
-; GFX908-NEXT: ds_add_f32 v2, v1
+; GFX908-NEXT: v_mov_b32_e32 v1, s0
+; GFX908-NEXT: v_mov_b32_e32 v2, s9
+; GFX908-NEXT: ds_add_f32 v1, v2
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: .LBB28_4:
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX908-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
-; GFX908-NEXT: v_add_f32_e32 v0, s10, v0
-; GFX908-NEXT: v_mov_b32_e32 v1, s10
+; GFX908-NEXT: v_add_f32_e32 v0, s8, v0
+; GFX908-NEXT: v_mov_b32_e32 v1, s8
; GFX908-NEXT: s_mov_b64 s[0:1], exec
; GFX908-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc
; GFX908-NEXT: v_bfrev_b32_e32 v1, 1
@@ -8680,48 +8708,52 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
;
; GFX8-LABEL: local_ds_fadd:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
; GFX8-NEXT: s_mov_b64 s[0:1], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX8-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX8-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX8-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_add_i32 s3, s3, 4
+; GFX8-NEXT: v_readfirstlane_b32 s6, v1
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_mov_b32 m0, -1
-; GFX8-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB28_2
; GFX8-NEXT: ; %bb.1:
-; GFX8-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX8-NEXT: s_lshl_b32 s8, s3, 3
-; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
-; GFX8-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, s8
-; GFX8-NEXT: ds_add_rtn_f32 v1, v2, v1
+; GFX8-NEXT: s_lshl_b32 s7, s3, 3
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_mov_b32_e32 v2, s6
+; GFX8-NEXT: ds_add_rtn_f32 v1, v1, v2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: .LBB28_2:
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_mov_b64 s[8:9], exec
-; GFX8-NEXT: v_readfirstlane_b32 s10, v1
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v1
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX8-NEXT: v_cvt_f32_i32_e32 v2, s0
+; GFX8-NEXT: v_readfirstlane_b32 s8, v1
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX8-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX8-NEXT: v_readfirstlane_b32 s9, v2
; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1
; GFX8-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
; GFX8-NEXT: s_cbranch_execz .LBB28_4
; GFX8-NEXT: ; %bb.3:
-; GFX8-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
-; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX8-NEXT: s_lshl_b32 s0, s3, 4
-; GFX8-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-NEXT: ds_add_f32 v2, v1
+; GFX8-NEXT: v_mov_b32_e32 v1, s0
+; GFX8-NEXT: v_mov_b32_e32 v2, s9
+; GFX8-NEXT: ds_add_f32 v1, v2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: .LBB28_4:
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX8-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
-; GFX8-NEXT: v_add_f32_e32 v0, s10, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s10
+; GFX8-NEXT: v_add_f32_e32 v0, s8, v0
+; GFX8-NEXT: v_mov_b32_e32 v1, s8
; GFX8-NEXT: s_mov_b64 s[0:1], exec
; GFX8-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc
; GFX8-NEXT: v_bfrev_b32_e32 v1, 1
@@ -8765,33 +8797,34 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
;
; GFX7-LABEL: local_ds_fadd:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2
; GFX7-NEXT: s_mov_b64 s[0:1], exec
-; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0
-; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0
+; GFX7-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_add_i32 s3, s3, 4
+; GFX7-NEXT: v_readfirstlane_b32 s10, v1
; GFX7-NEXT: ; implicit-def: $vgpr1
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX7-NEXT: s_cbranch_execz .LBB28_4
; GFX7-NEXT: ; %bb.1:
-; GFX7-NEXT: s_lshl_b32 s8, s3, 3
-; GFX7-NEXT: v_mov_b32_e32 v2, s8
+; GFX7-NEXT: s_lshl_b32 s0, s3, 3
+; GFX7-NEXT: v_mov_b32_e32 v2, s0
; GFX7-NEXT: ds_read_b32 v1, v2
-; GFX7-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v3, s0
-; GFX7-NEXT: v_mul_f32_e32 v3, 0x42280000, v3
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: .LBB28_2: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v4, v1
-; GFX7-NEXT: v_add_f32_e32 v1, v4, v3
-; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v2, v4, v1
+; GFX7-NEXT: v_mov_b32_e32 v3, v1
+; GFX7-NEXT: v_add_f32_e32 v1, s10, v3
+; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v2, v3, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v4
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v3
; GFX7-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB28_2
@@ -8799,30 +8832,31 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: .LBB28_4: ; %Flow23
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: s_mov_b64 s[8:9], exec
+; GFX7-NEXT: s_mov_b64 s[0:1], exec
+; GFX7-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7-NEXT: v_cvt_f32_i32_e32 v2, s0
; GFX7-NEXT: v_readfirstlane_b32 s10, v1
-; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v1, s8, 0
-; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v1, s9, v1
+; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX7-NEXT: v_readfirstlane_b32 s11, v2
; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1
; GFX7-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
; GFX7-NEXT: s_cbranch_execz .LBB28_7
; GFX7-NEXT: ; %bb.5:
; GFX7-NEXT: s_lshl_b32 s0, s3, 4
; GFX7-NEXT: v_mov_b32_e32 v1, s0
-; GFX7-NEXT: ds_read_b32 v3, v1
-; GFX7-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
-; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v2, s0
-; GFX7-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX7-NEXT: ds_read_b32 v2, v1
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: .LBB28_6: ; %atomicrmw.start2
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_add_f32_e32 v4, v3, v2
-; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
+; GFX7-NEXT: v_add_f32_e32 v3, s11, v2
+; GFX7-NEXT: ds_cmpst_rtn_b32 v3, v1, v2, v3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v4, v3
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v3, v2
; GFX7-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]
-; GFX7-NEXT: v_mov_b32_e32 v3, v4
+; GFX7-NEXT: v_mov_b32_e32 v2, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB28_6
; GFX7-NEXT: .LBB28_7: ; %Flow21
@@ -8889,33 +8923,34 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
;
; GFX6-LABEL: local_ds_fadd:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2
; GFX6-NEXT: s_mov_b64 s[0:1], exec
-; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0
-; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0
+; GFX6-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX6-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX6-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: s_add_i32 s3, s3, 4
+; GFX6-NEXT: v_readfirstlane_b32 s10, v1
; GFX6-NEXT: ; implicit-def: $vgpr1
; GFX6-NEXT: s_mov_b32 m0, -1
; GFX6-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX6-NEXT: s_cbranch_execz .LBB28_4
; GFX6-NEXT: ; %bb.1:
-; GFX6-NEXT: s_lshl_b32 s8, s3, 3
-; GFX6-NEXT: v_mov_b32_e32 v2, s8
+; GFX6-NEXT: s_lshl_b32 s0, s3, 3
+; GFX6-NEXT: v_mov_b32_e32 v2, s0
; GFX6-NEXT: ds_read_b32 v1, v2
-; GFX6-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v3, s0
-; GFX6-NEXT: v_mul_f32_e32 v3, 0x42280000, v3
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: .LBB28_2: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v4, v1
-; GFX6-NEXT: v_add_f32_e32 v1, v4, v3
-; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v2, v4, v1
+; GFX6-NEXT: v_mov_b32_e32 v3, v1
+; GFX6-NEXT: v_add_f32_e32 v1, s10, v3
+; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v2, v3, v1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v4
+; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v3
; GFX6-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB28_2
@@ -8923,30 +8958,31 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: .LBB28_4: ; %Flow21
; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX6-NEXT: s_mov_b64 s[8:9], exec
+; GFX6-NEXT: s_mov_b64 s[0:1], exec
+; GFX6-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s0
; GFX6-NEXT: v_readfirstlane_b32 s10, v1
-; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v1, s8, 0
-; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v1, s9, v1
+; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX6-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX6-NEXT: v_readfirstlane_b32 s11, v2
; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1
; GFX6-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
; GFX6-NEXT: s_cbranch_execz .LBB28_7
; GFX6-NEXT: ; %bb.5:
; GFX6-NEXT: s_lshl_b32 s0, s3, 4
; GFX6-NEXT: v_mov_b32_e32 v1, s0
-; GFX6-NEXT: ds_read_b32 v3, v1
-; GFX6-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
-; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v2, s0
-; GFX6-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX6-NEXT: ds_read_b32 v2, v1
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: .LBB28_6: ; %atomicrmw.start2
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_add_f32_e32 v4, v3, v2
-; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
+; GFX6-NEXT: v_add_f32_e32 v3, s11, v2
+; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v1, v2, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], v4, v3
+; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], v3, v2
; GFX6-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]
-; GFX6-NEXT: v_mov_b32_e32 v3, v4
+; GFX6-NEXT: v_mov_b32_e32 v2, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB28_6
; GFX6-NEXT: .LBB28_7: ; %Flow19
@@ -9026,45 +9062,49 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX12-LABEL: local_ds_fadd_one_as:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
-; GFX12-NEXT: s_mov_b32 s6, exec_lo
-; GFX12-NEXT: ; implicit-def: $vgpr1
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX12-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX12-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_mul_f32_e32 v1, 0x42280000, v0
+; GFX12-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX12-NEXT: v_readfirstlane_b32 s0, v1
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: ; implicit-def: $vgpr1
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_add_co_i32 s1, s3, 4
-; GFX12-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX12-NEXT: s_cbranch_execz .LBB29_2
+; GFX12-NEXT: s_and_saveexec_b32 s3, vcc_lo
; GFX12-NEXT: ; %bb.1:
-; GFX12-NEXT: s_bcnt1_i32_b32 s3, s6
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cvt_f32_ubyte0_e32 v1, s3
-; GFX12-NEXT: s_lshl_b32 s3, s1, 3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mul_f32 v1, 0x42280000, v1
-; GFX12-NEXT: ds_add_rtn_f32 v1, v2, v1
-; GFX12-NEXT: .LBB29_2:
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_lshl_b32 s6, s1, 3
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v1, s6
+; GFX12-NEXT: ds_add_rtn_f32 v1, v1, v2
+; GFX12-NEXT: ; %bb.2:
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_mov_b32 s7, exec_lo
+; GFX12-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-NEXT: v_mbcnt_lo_u32_b32 v3, exec_lo, 0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_bcnt1_i32_b32 s0, s0
; GFX12-NEXT: s_wait_dscnt 0x0
; GFX12-NEXT: v_readfirstlane_b32 s3, v1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mbcnt_lo_u32_b32 v2, s7, 0
+; GFX12-NEXT: v_cvt_f32_i32_e32 v2, s0
; GFX12-NEXT: s_mov_b32 s6, exec_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_cmpx_eq_u32_e32 0, v2
-; GFX12-NEXT: s_cbranch_execz .LBB29_4
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX12-NEXT: v_readfirstlane_b32 s7, v2
+; GFX12-NEXT: v_cmpx_eq_u32_e32 0, v3
; GFX12-NEXT: ; %bb.3:
-; GFX12-NEXT: s_bcnt1_i32_b32 s0, s7
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX12-NEXT: s_lshl_b32 s0, s1, 4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mul_f32 v1, 0x42280000, v1
-; GFX12-NEXT: ds_add_f32 v2, v1
-; GFX12-NEXT: .LBB29_4:
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-NEXT: v_dual_mov_b32 v2, s7 :: v_dual_mov_b32 v1, s0
+; GFX12-NEXT: ds_add_f32 v1, v2
+; GFX12-NEXT: ; %bb.4:
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s6
; GFX12-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX12-NEXT: s_mov_b32 s1, exec_lo
@@ -9115,46 +9155,48 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
;
; GFX942-LABEL: local_ds_fadd_one_as:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
; GFX942-NEXT: s_mov_b64 s[0:1], exec
-; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX942-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX942-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX942-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_add_i32 s3, s3, 4
+; GFX942-NEXT: v_readfirstlane_b32 s6, v1
; GFX942-NEXT: ; implicit-def: $vgpr1
-; GFX942-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB29_2
+; GFX942-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX942-NEXT: ; %bb.1:
+; GFX942-NEXT: s_lshl_b32 s7, s3, 3
+; GFX942-NEXT: v_mov_b32_e32 v1, s7
+; GFX942-NEXT: v_mov_b32_e32 v2, s6
+; GFX942-NEXT: ds_add_rtn_f32 v1, v1, v2
+; GFX942-NEXT: ; %bb.2:
+; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_mov_b64 s[0:1], exec
; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX942-NEXT: s_lshl_b32 s8, s3, 3
-; GFX942-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
-; GFX942-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX942-NEXT: v_mov_b32_e32 v2, s8
-; GFX942-NEXT: ds_add_rtn_f32 v1, v2, v1
-; GFX942-NEXT: .LBB29_2:
-; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX942-NEXT: s_mov_b64 s[8:9], exec
+; GFX942-NEXT: v_cvt_f32_i32_e32 v2, s0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_readfirstlane_b32 s10, v1
-; GFX942-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 0
-; GFX942-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v1
+; GFX942-NEXT: v_readfirstlane_b32 s8, v1
+; GFX942-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX942-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX942-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
; GFX942-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1
+; GFX942-NEXT: v_readfirstlane_b32 s9, v2
; GFX942-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
-; GFX942-NEXT: s_cbranch_execz .LBB29_4
; GFX942-NEXT: ; %bb.3:
-; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
-; GFX942-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX942-NEXT: s_lshl_b32 s0, s3, 4
-; GFX942-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX942-NEXT: v_mov_b32_e32 v2, s0
-; GFX942-NEXT: ds_add_f32 v2, v1
-; GFX942-NEXT: .LBB29_4:
+; GFX942-NEXT: v_mov_b32_e32 v1, s0
+; GFX942-NEXT: v_mov_b32_e32 v2, s9
+; GFX942-NEXT: ds_add_f32 v1, v2
+; GFX942-NEXT: ; %bb.4:
; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX942-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX942-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
-; GFX942-NEXT: v_add_f32_e32 v0, s10, v0
-; GFX942-NEXT: v_mov_b32_e32 v1, s10
+; GFX942-NEXT: v_add_f32_e32 v0, s8, v0
+; GFX942-NEXT: v_mov_b32_e32 v1, s8
; GFX942-NEXT: s_mov_b64 s[0:1], exec
; GFX942-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc
; GFX942-NEXT: v_bfrev_b32_e32 v1, 1
@@ -9196,42 +9238,45 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX11-LABEL: local_ds_fadd_one_as:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
-; GFX11-NEXT: s_mov_b32 s6, exec_lo
-; GFX11-NEXT: ; implicit-def: $vgpr1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX11-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX11-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_mul_f32_e32 v1, 0x42280000, v0
+; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX11-NEXT: v_readfirstlane_b32 s0, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX11-NEXT: ; implicit-def: $vgpr1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_add_i32 s1, s3, 4
-; GFX11-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX11-NEXT: s_cbranch_execz .LBB29_2
+; GFX11-NEXT: s_and_saveexec_b32 s3, vcc_lo
; GFX11-NEXT: ; %bb.1:
-; GFX11-NEXT: s_bcnt1_i32_b32 s3, s6
+; GFX11-NEXT: s_lshl_b32 s6, s1, 3
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s3
-; GFX11-NEXT: s_lshl_b32 s3, s1, 3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mul_f32 v1, 0x42280000, v1
-; GFX11-NEXT: ds_add_rtn_f32 v1, v2, v1
-; GFX11-NEXT: .LBB29_2:
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-NEXT: s_mov_b32 s7, exec_lo
+; GFX11-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v1, s6
+; GFX11-NEXT: ds_add_rtn_f32 v1, v1, v2
+; GFX11-NEXT: ; %bb.2:
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-NEXT: v_mbcnt_lo_u32_b32 v3, exec_lo, 0
+; GFX11-NEXT: s_bcnt1_i32_b32 s0, s0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_readfirstlane_b32 s3, v1
-; GFX11-NEXT: v_mbcnt_lo_u32_b32 v2, s7, 0
+; GFX11-NEXT: v_cvt_f32_i32_e32 v2, s0
; GFX11-NEXT: s_mov_b32 s6, exec_lo
-; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v2
-; GFX11-NEXT: s_cbranch_execz .LBB29_4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX11-NEXT: v_readfirstlane_b32 s7, v2
+; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v3
; GFX11-NEXT: ; %bb.3:
-; GFX11-NEXT: s_bcnt1_i32_b32 s0, s7
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX11-NEXT: s_lshl_b32 s0, s1, 4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mul_f32 v1, 0x42280000, v1
-; GFX11-NEXT: ds_add_f32 v2, v1
-; GFX11-NEXT: .LBB29_4:
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_dual_mov_b32 v2, s7 :: v_dual_mov_b32 v1, s0
+; GFX11-NEXT: ds_add_f32 v1, v2
+; GFX11-NEXT: ; %bb.4:
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s6
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX11-NEXT: v_bfrev_b32_e32 v1, 1
@@ -9278,39 +9323,41 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX10-LABEL: local_ds_fadd_one_as:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
-; GFX10-NEXT: ; implicit-def: $vgpr1
-; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX10-NEXT: s_mov_b32 s0, exec_lo
+; GFX10-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX10-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX10-NEXT: v_mul_f32_e32 v1, 0x42280000, v0
+; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10-NEXT: v_readfirstlane_b32 s0, v1
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-NEXT: ; implicit-def: $vgpr1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_add_i32 s1, s3, 4
-; GFX10-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX10-NEXT: s_cbranch_execz .LBB29_2
+; GFX10-NEXT: s_and_saveexec_b32 s3, vcc_lo
; GFX10-NEXT: ; %bb.1:
-; GFX10-NEXT: s_bcnt1_i32_b32 s3, s6
-; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s3
-; GFX10-NEXT: s_lshl_b32 s3, s1, 3
-; GFX10-NEXT: v_mov_b32_e32 v2, s3
-; GFX10-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX10-NEXT: ds_add_rtn_f32 v1, v2, v1
-; GFX10-NEXT: .LBB29_2:
+; GFX10-NEXT: s_lshl_b32 s6, s1, 3
+; GFX10-NEXT: v_mov_b32_e32 v2, s0
+; GFX10-NEXT: v_mov_b32_e32 v1, s6
+; GFX10-NEXT: ds_add_rtn_f32 v1, v1, v2
+; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX10-NEXT: s_mov_b32 s7, exec_lo
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX10-NEXT: s_mov_b32 s0, exec_lo
+; GFX10-NEXT: v_mbcnt_lo_u32_b32 v3, exec_lo, 0
+; GFX10-NEXT: s_bcnt1_i32_b32 s0, s0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_readfirstlane_b32 s3, v1
-; GFX10-NEXT: v_mbcnt_lo_u32_b32 v2, s7, 0
-; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v2
+; GFX10-NEXT: v_cvt_f32_i32_e32 v2, s0
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v3
+; GFX10-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v2
; GFX10-NEXT: s_and_saveexec_b32 s6, s0
-; GFX10-NEXT: s_cbranch_execz .LBB29_4
; GFX10-NEXT: ; %bb.3:
-; GFX10-NEXT: s_bcnt1_i32_b32 s0, s7
-; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX10-NEXT: s_lshl_b32 s0, s1, 4
-; GFX10-NEXT: v_mov_b32_e32 v2, s0
-; GFX10-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX10-NEXT: ds_add_f32 v2, v1
-; GFX10-NEXT: .LBB29_4:
+; GFX10-NEXT: v_mov_b32_e32 v2, s7
+; GFX10-NEXT: v_mov_b32_e32 v1, s0
+; GFX10-NEXT: ds_add_f32 v1, v2
+; GFX10-NEXT: ; %bb.4:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s6
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
@@ -9353,46 +9400,48 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
;
; GFX90A-LABEL: local_ds_fadd_one_as:
; GFX90A: ; %bb.0:
-; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
-; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX90A-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX90A-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: s_add_i32 s3, s3, 4
+; GFX90A-NEXT: v_readfirstlane_b32 s6, v1
; GFX90A-NEXT: ; implicit-def: $vgpr1
-; GFX90A-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GFX90A-NEXT: s_cbranch_execz .LBB29_2
+; GFX90A-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX90A-NEXT: ; %bb.1:
+; GFX90A-NEXT: s_lshl_b32 s7, s3, 3
+; GFX90A-NEXT: v_mov_b32_e32 v1, s7
+; GFX90A-NEXT: v_mov_b32_e32 v2, s6
+; GFX90A-NEXT: ds_add_rtn_f32 v1, v1, v2
+; GFX90A-NEXT: ; %bb.2:
+; GFX90A-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX90A-NEXT: s_mov_b64 s[0:1], exec
; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX90A-NEXT: s_lshl_b32 s8, s3, 3
-; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
-; GFX90A-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX90A-NEXT: v_mov_b32_e32 v2, s8
-; GFX90A-NEXT: ds_add_rtn_f32 v1, v2, v1
-; GFX90A-NEXT: .LBB29_2:
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_mov_b64 s[8:9], exec
+; GFX90A-NEXT: v_cvt_f32_i32_e32 v2, s0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_readfirstlane_b32 s10, v1
-; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 0
-; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v1
+; GFX90A-NEXT: v_readfirstlane_b32 s8, v1
+; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX90A-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX90A-NEXT: v_readfirstlane_b32 s9, v2
; GFX90A-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1
; GFX90A-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
-; GFX90A-NEXT: s_cbranch_execz .LBB29_4
; GFX90A-NEXT: ; %bb.3:
-; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
-; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX90A-NEXT: s_lshl_b32 s0, s3, 4
-; GFX90A-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX90A-NEXT: v_mov_b32_e32 v2, s0
-; GFX90A-NEXT: ds_add_f32 v2, v1
-; GFX90A-NEXT: .LBB29_4:
+; GFX90A-NEXT: v_mov_b32_e32 v1, s0
+; GFX90A-NEXT: v_mov_b32_e32 v2, s9
+; GFX90A-NEXT: ds_add_f32 v1, v2
+; GFX90A-NEXT: ; %bb.4:
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX90A-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
-; GFX90A-NEXT: v_add_f32_e32 v0, s10, v0
-; GFX90A-NEXT: v_mov_b32_e32 v1, s10
+; GFX90A-NEXT: v_add_f32_e32 v0, s8, v0
+; GFX90A-NEXT: v_mov_b32_e32 v1, s8
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
; GFX90A-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc
; GFX90A-NEXT: v_bfrev_b32_e32 v1, 1
@@ -9432,46 +9481,48 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
;
; GFX908-LABEL: local_ds_fadd_one_as:
; GFX908: ; %bb.0:
-; GFX908-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
; GFX908-NEXT: s_mov_b64 s[0:1], exec
-; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX908-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX908-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX908-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX908-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_add_i32 s3, s3, 4
+; GFX908-NEXT: v_readfirstlane_b32 s6, v1
; GFX908-NEXT: ; implicit-def: $vgpr1
-; GFX908-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GFX908-NEXT: s_cbranch_execz .LBB29_2
+; GFX908-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX908-NEXT: ; %bb.1:
+; GFX908-NEXT: s_lshl_b32 s7, s3, 3
+; GFX908-NEXT: v_mov_b32_e32 v1, s7
+; GFX908-NEXT: v_mov_b32_e32 v2, s6
+; GFX908-NEXT: ds_add_rtn_f32 v1, v1, v2
+; GFX908-NEXT: ; %bb.2:
+; GFX908-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX908-NEXT: s_mov_b64 s[0:1], exec
; GFX908-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX908-NEXT: s_lshl_b32 s8, s3, 3
-; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
-; GFX908-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX908-NEXT: v_mov_b32_e32 v2, s8
-; GFX908-NEXT: ds_add_rtn_f32 v1, v2, v1
-; GFX908-NEXT: .LBB29_2:
-; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_mov_b64 s[8:9], exec
+; GFX908-NEXT: v_cvt_f32_i32_e32 v2, s0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_readfirstlane_b32 s10, v1
-; GFX908-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 0
-; GFX908-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v1
+; GFX908-NEXT: v_readfirstlane_b32 s8, v1
+; GFX908-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX908-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX908-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX908-NEXT: v_readfirstlane_b32 s9, v2
; GFX908-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1
; GFX908-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
-; GFX908-NEXT: s_cbranch_execz .LBB29_4
; GFX908-NEXT: ; %bb.3:
-; GFX908-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
-; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX908-NEXT: s_lshl_b32 s0, s3, 4
-; GFX908-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX908-NEXT: v_mov_b32_e32 v2, s0
-; GFX908-NEXT: ds_add_f32 v2, v1
-; GFX908-NEXT: .LBB29_4:
+; GFX908-NEXT: v_mov_b32_e32 v1, s0
+; GFX908-NEXT: v_mov_b32_e32 v2, s9
+; GFX908-NEXT: ds_add_f32 v1, v2
+; GFX908-NEXT: ; %bb.4:
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX908-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
-; GFX908-NEXT: v_add_f32_e32 v0, s10, v0
-; GFX908-NEXT: v_mov_b32_e32 v1, s10
+; GFX908-NEXT: v_add_f32_e32 v0, s8, v0
+; GFX908-NEXT: v_mov_b32_e32 v1, s8
; GFX908-NEXT: s_mov_b64 s[0:1], exec
; GFX908-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc
; GFX908-NEXT: v_bfrev_b32_e32 v1, 1
@@ -9511,47 +9562,49 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
;
; GFX8-LABEL: local_ds_fadd_one_as:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
; GFX8-NEXT: s_mov_b64 s[0:1], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX8-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX8-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX8-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_add_i32 s3, s3, 4
+; GFX8-NEXT: v_readfirstlane_b32 s6, v1
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_mov_b32 m0, -1
-; GFX8-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GFX8-NEXT: s_cbranch_execz .LBB29_2
+; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: ; %bb.1:
+; GFX8-NEXT: s_lshl_b32 s7, s3, 3
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_mov_b32_e32 v2, s6
+; GFX8-NEXT: ds_add_rtn_f32 v1, v1, v2
+; GFX8-NEXT: ; %bb.2:
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
; GFX8-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX8-NEXT: s_lshl_b32 s8, s3, 3
-; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
-; GFX8-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, s8
-; GFX8-NEXT: ds_add_rtn_f32 v1, v2, v1
-; GFX8-NEXT: .LBB29_2:
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_mov_b64 s[8:9], exec
+; GFX8-NEXT: v_cvt_f32_i32_e32 v2, s0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_readfirstlane_b32 s10, v1
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v1
+; GFX8-NEXT: v_readfirstlane_b32 s8, v1
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX8-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX8-NEXT: v_readfirstlane_b32 s9, v2
; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1
; GFX8-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
-; GFX8-NEXT: s_cbranch_execz .LBB29_4
; GFX8-NEXT: ; %bb.3:
-; GFX8-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
-; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX8-NEXT: s_lshl_b32 s0, s3, 4
-; GFX8-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-NEXT: ds_add_f32 v2, v1
-; GFX8-NEXT: .LBB29_4:
+; GFX8-NEXT: v_mov_b32_e32 v1, s0
+; GFX8-NEXT: v_mov_b32_e32 v2, s9
+; GFX8-NEXT: ds_add_f32 v1, v2
+; GFX8-NEXT: ; %bb.4:
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX8-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
-; GFX8-NEXT: v_add_f32_e32 v0, s10, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s10
+; GFX8-NEXT: v_add_f32_e32 v0, s8, v0
+; GFX8-NEXT: v_mov_b32_e32 v1, s8
; GFX8-NEXT: s_mov_b64 s[0:1], exec
; GFX8-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc
; GFX8-NEXT: v_bfrev_b32_e32 v1, 1
@@ -9593,33 +9646,34 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
;
; GFX7-LABEL: local_ds_fadd_one_as:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2
; GFX7-NEXT: s_mov_b64 s[0:1], exec
-; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0
-; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0
+; GFX7-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_add_i32 s3, s3, 4
+; GFX7-NEXT: v_readfirstlane_b32 s10, v1
; GFX7-NEXT: ; implicit-def: $vgpr1
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX7-NEXT: s_cbranch_execz .LBB29_4
; GFX7-NEXT: ; %bb.1:
-; GFX7-NEXT: s_lshl_b32 s8, s3, 3
-; GFX7-NEXT: v_mov_b32_e32 v2, s8
+; GFX7-NEXT: s_lshl_b32 s0, s3, 3
+; GFX7-NEXT: v_mov_b32_e32 v2, s0
; GFX7-NEXT: ds_read_b32 v1, v2
-; GFX7-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v3, s0
-; GFX7-NEXT: v_mul_f32_e32 v3, 0x42280000, v3
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: .LBB29_2: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v4, v1
-; GFX7-NEXT: v_add_f32_e32 v1, v4, v3
-; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v2, v4, v1
+; GFX7-NEXT: v_mov_b32_e32 v3, v1
+; GFX7-NEXT: v_add_f32_e32 v1, s10, v3
+; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v2, v3, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v4
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v3
; GFX7-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB29_2
@@ -9627,30 +9681,31 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: .LBB29_4: ; %Flow23
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: s_mov_b64 s[8:9], exec
+; GFX7-NEXT: s_mov_b64 s[0:1], exec
+; GFX7-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7-NEXT: v_cvt_f32_i32_e32 v2, s0
; GFX7-NEXT: v_readfirstlane_b32 s10, v1
-; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v1, s8, 0
-; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v1, s9, v1
+; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX7-NEXT: v_readfirstlane_b32 s11, v2
; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1
; GFX7-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
; GFX7-NEXT: s_cbranch_execz .LBB29_7
; GFX7-NEXT: ; %bb.5:
; GFX7-NEXT: s_lshl_b32 s0, s3, 4
; GFX7-NEXT: v_mov_b32_e32 v1, s0
-; GFX7-NEXT: ds_read_b32 v3, v1
-; GFX7-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
-; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v2, s0
-; GFX7-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX7-NEXT: ds_read_b32 v2, v1
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: .LBB29_6: ; %atomicrmw.start2
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_add_f32_e32 v4, v3, v2
-; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
+; GFX7-NEXT: v_add_f32_e32 v3, s11, v2
+; GFX7-NEXT: ds_cmpst_rtn_b32 v3, v1, v2, v3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v4, v3
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v3, v2
; GFX7-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]
-; GFX7-NEXT: v_mov_b32_e32 v3, v4
+; GFX7-NEXT: v_mov_b32_e32 v2, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB29_6
; GFX7-NEXT: .LBB29_7: ; %Flow21
@@ -9717,33 +9772,34 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
;
; GFX6-LABEL: local_ds_fadd_one_as:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2
; GFX6-NEXT: s_mov_b64 s[0:1], exec
-; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0
-; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0
+; GFX6-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX6-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX6-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: s_add_i32 s3, s3, 4
+; GFX6-NEXT: v_readfirstlane_b32 s10, v1
; GFX6-NEXT: ; implicit-def: $vgpr1
; GFX6-NEXT: s_mov_b32 m0, -1
; GFX6-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX6-NEXT: s_cbranch_execz .LBB29_4
; GFX6-NEXT: ; %bb.1:
-; GFX6-NEXT: s_lshl_b32 s8, s3, 3
-; GFX6-NEXT: v_mov_b32_e32 v2, s8
+; GFX6-NEXT: s_lshl_b32 s0, s3, 3
+; GFX6-NEXT: v_mov_b32_e32 v2, s0
; GFX6-NEXT: ds_read_b32 v1, v2
-; GFX6-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v3, s0
-; GFX6-NEXT: v_mul_f32_e32 v3, 0x42280000, v3
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: .LBB29_2: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v4, v1
-; GFX6-NEXT: v_add_f32_e32 v1, v4, v3
-; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v2, v4, v1
+; GFX6-NEXT: v_mov_b32_e32 v3, v1
+; GFX6-NEXT: v_add_f32_e32 v1, s10, v3
+; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v2, v3, v1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v4
+; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v3
; GFX6-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB29_2
@@ -9751,30 +9807,31 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: .LBB29_4: ; %Flow21
; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX6-NEXT: s_mov_b64 s[8:9], exec
+; GFX6-NEXT: s_mov_b64 s[0:1], exec
+; GFX6-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s0
; GFX6-NEXT: v_readfirstlane_b32 s10, v1
-; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v1, s8, 0
-; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v1, s9, v1
+; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX6-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX6-NEXT: v_readfirstlane_b32 s11, v2
; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1
; GFX6-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
; GFX6-NEXT: s_cbranch_execz .LBB29_7
; GFX6-NEXT: ; %bb.5:
; GFX6-NEXT: s_lshl_b32 s0, s3, 4
; GFX6-NEXT: v_mov_b32_e32 v1, s0
-; GFX6-NEXT: ds_read_b32 v3, v1
-; GFX6-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
-; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v2, s0
-; GFX6-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX6-NEXT: ds_read_b32 v2, v1
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: .LBB29_6: ; %atomicrmw.start2
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_add_f32_e32 v4, v3, v2
-; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
+; GFX6-NEXT: v_add_f32_e32 v3, s11, v2
+; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v1, v2, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], v4, v3
+; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], v3, v2
; GFX6-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]
-; GFX6-NEXT: v_mov_b32_e32 v3, v4
+; GFX6-NEXT: v_mov_b32_e32 v2, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB29_6
; GFX6-NEXT: .LBB29_7: ; %Flow19
>From da8dad2839b628d692c80ddae8d83c73321dd307 Mon Sep 17 00:00:00 2001
From: Aaditya <Aaditya.AlokDeshpande at amd.com>
Date: Mon, 27 Jul 2026 15:26:04 +0530
Subject: [PATCH 2/3] Use unsigned intrinsics + code formatting.
---
llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp | 4 +++-
llvm/test/CodeGen/AMDGPU/global-atomic-scan.ll | 16 ++++++++--------
2 files changed, 11 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
index fffd01efca39b..946b03b6fb9d5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
@@ -660,11 +660,13 @@ static Intrinsic::ID getWaveReductionIntrinsic(AtomicRMWInst::BinOp Op) {
case AtomicRMWInst::Xor:
return Intrinsic::amdgcn_wave_reduce_xor;
case AtomicRMWInst::UMax:
+ return Intrinsic::amdgcn_wave_reduce_umax;
case AtomicRMWInst::Max:
return Intrinsic::amdgcn_wave_reduce_max;
case AtomicRMWInst::FMax:
return Intrinsic::amdgcn_wave_reduce_fmax;
case AtomicRMWInst::UMin:
+ return Intrinsic::amdgcn_wave_reduce_umin;
case AtomicRMWInst::Min:
return Intrinsic::amdgcn_wave_reduce_min;
case AtomicRMWInst::FMin:
@@ -782,7 +784,7 @@ void AMDGPUAtomicOptimizerImpl::optimizeAtomic(Instruction &I,
BasicBlock *ComputeEnd = nullptr;
if (UseWaveReductionIntrinsic) {
// Build reductions with wave-reduce intrinsics.
- unsigned Strategy = (ScanImpl == ScanOptions::DPP) ? 2 : 1;
+ unsigned Strategy = ScanImpl == ScanOptions::DPP ? 2 : 1;
Intrinsic::ID WaveRedIntrinsic = getWaveReductionIntrinsic(Op);
NewV = B.CreateIntrinsic(WaveRedIntrinsic, Ty, {V, B.getInt32(Strategy)});
} else {
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomic-scan.ll b/llvm/test/CodeGen/AMDGPU/global-atomic-scan.ll
index c9469d393d2ea..c508d1adfa0c5 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomic-scan.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomic-scan.ll
@@ -987,7 +987,7 @@ define amdgpu_kernel void @atomic_umax_i32_offset(ptr addrspace(1) %out, i32 %in
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.max.i32(i32 [[IN1:%.*]], i32 1)
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.umax.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
; IR: 8:
@@ -1012,7 +1012,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret_offset(ptr addrspace(1) %out, ptr
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.max.i32(i32 [[IN1:%.*]], i32 1)
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.umax.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
; IR: 8:
@@ -1045,7 +1045,7 @@ define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr addrspace(1) %out,
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.max.i32(i32 [[IN1:%.*]], i32 1)
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.umax.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
; IR: 8:
@@ -1072,7 +1072,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64_offset(ptr addrspace(1) %o
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.max.i32(i32 [[IN1:%.*]], i32 1)
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.umax.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
; IR: 8:
@@ -1104,7 +1104,7 @@ define amdgpu_kernel void @atomic_umax_i32(ptr addrspace(1) %out, i32 %in) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.max.i32(i32 [[IN1:%.*]], i32 1)
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.umax.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
; IR: 8:
@@ -1127,7 +1127,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret(ptr addrspace(1) %out, ptr addrsp
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.max.i32(i32 [[IN1:%.*]], i32 1)
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.umax.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
; IR: 8:
@@ -1158,7 +1158,7 @@ define amdgpu_kernel void @atomic_umax_i32_addr64(ptr addrspace(1) %out, i32 %in
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.max.i32(i32 [[IN1:%.*]], i32 1)
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.umax.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
; IR: 8:
@@ -1183,7 +1183,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64(ptr addrspace(1) %out, ptr
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.max.i32(i32 [[IN1:%.*]], i32 1)
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.umax.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
; IR: 8:
>From f7d75bff489e8efae2d7d5c42b0fa9326b4b75b7 Mon Sep 17 00:00:00 2001
From: Aaditya <Aaditya.AlokDeshpande at amd.com>
Date: Wed, 29 Jul 2026 13:43:04 +0530
Subject: [PATCH 3/3] Add missed optimization: Remove redundant S_MUL instrs
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 68 ++-
.../atomic_optimizations_mul_one.ll | 396 ++++++++++++++++--
2 files changed, 432 insertions(+), 32 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 77b3f3ef3ef4a..52e2cd450b4ca 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -5972,11 +5972,34 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
// parity the result will be the same as the input value.
Register ParityRegister =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
-
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_AND_B32), ParityRegister)
.addReg(NewAccumulator->getOperand(0).getReg())
.addImm(1)
.setOperandDead(3); // Dead scc
+ // Check if Src is a known identity constant.
+ MachineInstr *SrcDef = MRI.getVRegDef(SrcReg);
+ if (SrcDef && SrcDef->isMoveImmediate()) {
+ int64_t Imm = SrcDef->getOperand(1).getImm();
+ unsigned XorMovOp = Opc == AMDGPU::S_XOR_B32
+ ? AMDGPU::S_MOV_B32
+ : AMDGPU::S_MOV_B64_IMM_PSEUDO;
+ if (Imm == 0) { // 0 * parity(exec) = 0
+ BuildMI(BB, MI, DL, TII->get(XorMovOp), DstReg).addImm(0);
+ break;
+ }
+ if (Imm == 1) { // 1 * parity(exec) = parity(exec)
+ if (Opc == AMDGPU::S_XOR_B32) {
+ BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MOV_B32), DstReg)
+ .addReg(ParityRegister);
+ } else {
+ Register DstHi =
+ MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
+ BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MOV_B32), DstHi).addImm(0);
+ BuildRegSequence(BB, MI, DstReg, ParityRegister, DstHi);
+ }
+ break;
+ }
+ }
if (Opc == AMDGPU::S_XOR_B32) {
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DstReg)
.addReg(SrcReg)
@@ -6000,7 +6023,15 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
}
case AMDGPU::S_SUB_I32: {
Register NegatedVal = MRI.createVirtualRegister(DstRegClass);
-
+ // Check if Src is a known identity constant.
+ MachineInstr *SrcDef = MRI.getVRegDef(SrcReg);
+ if (SrcDef && SrcDef->isMoveImmediate()) {
+ int64_t Imm = SrcDef->getOperand(1).getImm();
+ if (Imm == 0) { // 0 * bitcount(exec) = 0
+ BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MOV_B32), DstReg).addImm(0);
+ break;
+ }
+ }
// Take the negation of the source operand.
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_SUB_I32), NegatedVal)
.addImm(0)
@@ -6011,6 +6042,20 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
break;
}
case AMDGPU::S_ADD_I32: {
+ // Check if Src is a known identity constant.
+ MachineInstr *SrcDef = MRI.getVRegDef(SrcReg);
+ if (SrcDef && SrcDef->isMoveImmediate()) {
+ int64_t Imm = SrcDef->getOperand(1).getImm();
+ if (Imm == 0) { // 0 * bitcount(exec) = 0
+ BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MOV_B32), DstReg).addImm(0);
+ break;
+ }
+ if (Imm == 1) { // 1 * bitcount(exec) = bitcount(exec)
+ BuildMI(BB, MI, DL, TII->get(AMDGPU::COPY), DstReg)
+ .addReg(NewAccumulator->getOperand(0).getReg());
+ break;
+ }
+ }
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DstReg)
.addReg(SrcReg)
.addReg(NewAccumulator->getOperand(0).getReg());
@@ -6033,6 +6078,25 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
auto [Op1L, Op1H] = ExtractSubRegs(MI, MI.getOperand(1),
MRI.getRegClass(SrcReg), ST, MRI);
+ // Check if Src is a known identity constant.
+ MachineInstr *SrcDef = MRI.getVRegDef(SrcReg);
+ if (SrcDef && SrcDef->isMoveImmediate()) {
+ int64_t Imm = SrcDef->getOperand(1).getImm();
+ if (Imm == 0) { // 0 * bitcount(exec) = 0
+ BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MOV_B64_IMM_PSEUDO), DstReg)
+ .addImm(0);
+ break;
+ }
+ if (Imm == 1 && Opc == AMDGPU::S_ADD_U64_PSEUDO) {
+ // 1 * bitcount(exec) = bitcount(exec)
+ Register DstHi =
+ MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
+ BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MOV_B32), DstHi).addImm(0);
+ BuildRegSequence(BB, MI, DstReg,
+ NewAccumulator->getOperand(0).getReg(), DstHi);
+ break;
+ }
+ }
if (Opc == AMDGPU::S_SUB_U64_PSEUDO) {
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_SUB_I32), NegatedValLo)
.addImm(0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll
index 760b0a1a1a2c8..df06784c56b1b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll
@@ -42,7 +42,6 @@ define amdgpu_cs void @atomic_add(<4 x i32> inreg %arg) {
; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GCN-NEXT: s_cbranch_execz .LBB0_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_mul_i32 s4, s4, 1
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: buffer_atomic_add v0, v1, s[0:3], 0 idxen
@@ -87,7 +86,6 @@ define amdgpu_cs void @atomic_add_and_format(<4 x i32> inreg %arg) {
; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN-NEXT: s_cbranch_execz .LBB1_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_mul_i32 s6, s6, 1
; GCN-NEXT: v_mov_b32_e32 v1, s6
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: buffer_atomic_add v1, v2, s[0:3], 0 idxen glc
@@ -138,7 +136,6 @@ define amdgpu_cs void @atomic_sub(<4 x i32> inreg %arg) {
; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GCN-NEXT: s_cbranch_execz .LBB2_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_mul_i32 s4, s4, 1
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: buffer_atomic_sub v0, v1, s[0:3], 0 idxen
@@ -149,6 +146,237 @@ define amdgpu_cs void @atomic_sub(<4 x i32> inreg %arg) {
ret void
}
+define amdgpu_cs void @atomic_add_constant_0(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_add_constant_0(
+; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
+; IR-NEXT: [[_ENTRY:.*:]]
+; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
+; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
+; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 0, i32 1)
+; IR-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]
+; IR: [[BB8]]:
+; IR-NEXT: [[TMP9:%.*]] = call i32 @llvm.amdgcn.struct.buffer.atomic.add.i32(i32 [[TMP6]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB10]]
+; IR: [[BB10]]:
+; IR-NEXT: ret void
+;
+; GCN-LABEL: atomic_add_constant_0:
+; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT: s_cbranch_execz .LBB3_2
+; GCN-NEXT: ; %bb.1:
+; GCN-NEXT: v_mov_b32_e32 v0, 0
+; GCN-NEXT: buffer_atomic_add v0, v0, s[0:3], 0 idxen
+; GCN-NEXT: .LBB3_2:
+; GCN-NEXT: s_endpgm
+.entry:
+ call i32 @llvm.amdgcn.struct.buffer.atomic.add.i32(i32 0, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
+ ret void
+}
+
+define amdgpu_cs void @atomic_sub_constant_0(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_sub_constant_0(
+; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
+; IR-NEXT: [[_ENTRY:.*:]]
+; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
+; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
+; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 0, i32 1)
+; IR-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]
+; IR: [[BB8]]:
+; IR-NEXT: [[TMP9:%.*]] = call i32 @llvm.amdgcn.struct.buffer.atomic.sub.i32(i32 [[TMP6]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB10]]
+; IR: [[BB10]]:
+; IR-NEXT: ret void
+;
+; GCN-LABEL: atomic_sub_constant_0:
+; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT: s_cbranch_execz .LBB4_2
+; GCN-NEXT: ; %bb.1:
+; GCN-NEXT: v_mov_b32_e32 v0, 0
+; GCN-NEXT: buffer_atomic_sub v0, v0, s[0:3], 0 idxen
+; GCN-NEXT: .LBB4_2:
+; GCN-NEXT: s_endpgm
+.entry:
+ call i32 @llvm.amdgcn.struct.buffer.atomic.sub.i32(i32 0, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
+ ret void
+}
+
+
+define amdgpu_cs void @atomic_add_i64_constant_1(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_add_i64_constant_1(
+; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
+; IR-NEXT: [[_ENTRY:.*:]]
+; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
+; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
+; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.amdgcn.wave.reduce.add.i64(i64 1, i32 1)
+; IR-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]
+; IR: [[BB8]]:
+; IR-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.struct.buffer.atomic.add.i64(i64 [[TMP6]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB10]]
+; IR: [[BB10]]:
+; IR-NEXT: ret void
+;
+; GCN-LABEL: atomic_add_i64_constant_1:
+; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GCN-NEXT: s_mov_b64 s[6:7], exec
+; GCN-NEXT: s_mov_b32 s5, 0
+; GCN-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GCN-NEXT: s_cbranch_execz .LBB5_2
+; GCN-NEXT: ; %bb.1:
+; GCN-NEXT: v_mov_b32_e32 v0, s4
+; GCN-NEXT: v_mov_b32_e32 v1, s5
+; GCN-NEXT: v_mov_b32_e32 v2, 0
+; GCN-NEXT: buffer_atomic_add_x2 v[0:1], v2, s[0:3], 0 idxen
+; GCN-NEXT: .LBB5_2:
+; GCN-NEXT: s_endpgm
+.entry:
+ call i64 @llvm.amdgcn.struct.buffer.atomic.add.i64(i64 1, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
+ ret void
+}
+
+define amdgpu_cs void @atomic_add_i64_constant_0(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_add_i64_constant_0(
+; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
+; IR-NEXT: [[_ENTRY:.*:]]
+; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
+; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
+; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.amdgcn.wave.reduce.add.i64(i64 0, i32 1)
+; IR-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]
+; IR: [[BB8]]:
+; IR-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.struct.buffer.atomic.add.i64(i64 [[TMP6]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB10]]
+; IR: [[BB10]]:
+; IR-NEXT: ret void
+;
+; GCN-LABEL: atomic_add_i64_constant_0:
+; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT: s_cbranch_execz .LBB6_2
+; GCN-NEXT: ; %bb.1:
+; GCN-NEXT: v_mov_b32_e32 v0, 0
+; GCN-NEXT: v_mov_b32_e32 v1, 0
+; GCN-NEXT: v_mov_b32_e32 v2, 0
+; GCN-NEXT: buffer_atomic_add_x2 v[0:1], v2, s[0:3], 0 idxen
+; GCN-NEXT: .LBB6_2:
+; GCN-NEXT: s_endpgm
+.entry:
+ call i64 @llvm.amdgcn.struct.buffer.atomic.add.i64(i64 0, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
+ ret void
+}
+
+define amdgpu_cs void @atomic_sub_i64_constant_1(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_sub_i64_constant_1(
+; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
+; IR-NEXT: [[_ENTRY:.*:]]
+; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
+; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
+; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.amdgcn.wave.reduce.add.i64(i64 1, i32 1)
+; IR-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]
+; IR: [[BB8]]:
+; IR-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.struct.buffer.atomic.sub.i64(i64 [[TMP6]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB10]]
+; IR: [[BB10]]:
+; IR-NEXT: ret void
+;
+; GCN-LABEL: atomic_sub_i64_constant_1:
+; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GCN-NEXT: s_mov_b64 s[6:7], exec
+; GCN-NEXT: s_mov_b32 s5, 0
+; GCN-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GCN-NEXT: s_cbranch_execz .LBB7_2
+; GCN-NEXT: ; %bb.1:
+; GCN-NEXT: v_mov_b32_e32 v0, s4
+; GCN-NEXT: v_mov_b32_e32 v1, s5
+; GCN-NEXT: v_mov_b32_e32 v2, 0
+; GCN-NEXT: buffer_atomic_sub_x2 v[0:1], v2, s[0:3], 0 idxen
+; GCN-NEXT: .LBB7_2:
+; GCN-NEXT: s_endpgm
+.entry:
+ call i64 @llvm.amdgcn.struct.buffer.atomic.sub.i64(i64 1, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
+ ret void
+}
+
+define amdgpu_cs void @atomic_sub_i64_constant_0(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_sub_i64_constant_0(
+; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
+; IR-NEXT: [[_ENTRY:.*:]]
+; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
+; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
+; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.amdgcn.wave.reduce.add.i64(i64 0, i32 1)
+; IR-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]
+; IR: [[BB8]]:
+; IR-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.struct.buffer.atomic.sub.i64(i64 [[TMP6]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB10]]
+; IR: [[BB10]]:
+; IR-NEXT: ret void
+;
+; GCN-LABEL: atomic_sub_i64_constant_0:
+; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT: s_cbranch_execz .LBB8_2
+; GCN-NEXT: ; %bb.1:
+; GCN-NEXT: v_mov_b32_e32 v0, 0
+; GCN-NEXT: v_mov_b32_e32 v1, 0
+; GCN-NEXT: v_mov_b32_e32 v2, 0
+; GCN-NEXT: buffer_atomic_sub_x2 v[0:1], v2, s[0:3], 0 idxen
+; GCN-NEXT: .LBB8_2:
+; GCN-NEXT: s_endpgm
+.entry:
+ call i64 @llvm.amdgcn.struct.buffer.atomic.sub.i64(i64 0, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
+ ret void
+}
+
define amdgpu_cs void @atomic_sub_and_format(<4 x i32> inreg %arg) {
; IR-LABEL: define amdgpu_cs void @atomic_sub_and_format(
; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
@@ -181,13 +409,12 @@ define amdgpu_cs void @atomic_sub_and_format(<4 x i32> inreg %arg) {
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_cbranch_execz .LBB3_2
+; GCN-NEXT: s_cbranch_execz .LBB9_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_mul_i32 s6, s6, 1
; GCN-NEXT: v_mov_b32_e32 v1, s6
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: buffer_atomic_sub v1, v2, s[0:3], 0 idxen glc
-; GCN-NEXT: .LBB3_2:
+; GCN-NEXT: .LBB9_2:
; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_readfirstlane_b32 s4, v1
@@ -232,20 +459,136 @@ define amdgpu_cs void @atomic_xor(<4 x i32> inreg %arg) {
; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GCN-NEXT: s_cbranch_execz .LBB4_2
+; GCN-NEXT: s_cbranch_execz .LBB10_2
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_and_b32 s4, s4, 1
-; GCN-NEXT: s_mul_i32 s4, s4, 1
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: buffer_atomic_xor v0, v1, s[0:3], 0 idxen
-; GCN-NEXT: .LBB4_2:
+; GCN-NEXT: .LBB10_2:
; GCN-NEXT: s_endpgm
.entry:
call i32 @llvm.amdgcn.struct.buffer.atomic.xor.i32(i32 1, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
ret void
}
+define amdgpu_cs void @atomic_xor_constant_0(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_xor_constant_0(
+; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
+; IR-NEXT: [[_ENTRY:.*:]]
+; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
+; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
+; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.wave.reduce.xor.i32(i32 0, i32 1)
+; IR-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]
+; IR: [[BB8]]:
+; IR-NEXT: [[TMP9:%.*]] = call i32 @llvm.amdgcn.struct.buffer.atomic.xor.i32(i32 [[TMP6]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB10]]
+; IR: [[BB10]]:
+; IR-NEXT: ret void
+;
+; GCN-LABEL: atomic_xor_constant_0:
+; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT: s_cbranch_execz .LBB11_2
+; GCN-NEXT: ; %bb.1:
+; GCN-NEXT: v_mov_b32_e32 v0, 0
+; GCN-NEXT: buffer_atomic_xor v0, v0, s[0:3], 0 idxen
+; GCN-NEXT: .LBB11_2:
+; GCN-NEXT: s_endpgm
+.entry:
+ call i32 @llvm.amdgcn.struct.buffer.atomic.xor.i32(i32 0, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
+ ret void
+}
+
+define amdgpu_cs void @atomic_xor_constant_0_i64(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_xor_constant_0_i64(
+; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
+; IR-NEXT: [[_ENTRY:.*:]]
+; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
+; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
+; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.amdgcn.wave.reduce.xor.i64(i64 0, i32 1)
+; IR-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]
+; IR: [[BB8]]:
+; IR-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.struct.buffer.atomic.xor.i64(i64 [[TMP6]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB10]]
+; IR: [[BB10]]:
+; IR-NEXT: ret void
+;
+; GCN-LABEL: atomic_xor_constant_0_i64:
+; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT: s_cbranch_execz .LBB12_2
+; GCN-NEXT: ; %bb.1:
+; GCN-NEXT: v_mov_b32_e32 v0, 0
+; GCN-NEXT: v_mov_b32_e32 v1, 0
+; GCN-NEXT: v_mov_b32_e32 v2, 0
+; GCN-NEXT: buffer_atomic_xor_x2 v[0:1], v2, s[0:3], 0 idxen
+; GCN-NEXT: .LBB12_2:
+; GCN-NEXT: s_endpgm
+.entry:
+ call i64 @llvm.amdgcn.struct.buffer.atomic.xor.i64(i64 0, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
+ ret void
+}
+
+define amdgpu_cs void @atomic_xor_constant_1_i64(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_xor_constant_1_i64(
+; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
+; IR-NEXT: [[_ENTRY:.*:]]
+; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
+; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
+; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.amdgcn.wave.reduce.xor.i64(i64 1, i32 1)
+; IR-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]
+; IR: [[BB8]]:
+; IR-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.struct.buffer.atomic.xor.i64(i64 [[TMP6]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB10]]
+; IR: [[BB10]]:
+; IR-NEXT: ret void
+;
+; GCN-LABEL: atomic_xor_constant_1_i64:
+; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GCN-NEXT: s_mov_b64 s[6:7], exec
+; GCN-NEXT: s_mov_b32 s5, 0
+; GCN-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GCN-NEXT: s_cbranch_execz .LBB13_2
+; GCN-NEXT: ; %bb.1:
+; GCN-NEXT: s_and_b32 s4, s4, 1
+; GCN-NEXT: v_mov_b32_e32 v0, s4
+; GCN-NEXT: v_mov_b32_e32 v1, s5
+; GCN-NEXT: v_mov_b32_e32 v2, 0
+; GCN-NEXT: buffer_atomic_xor_x2 v[0:1], v2, s[0:3], 0 idxen
+; GCN-NEXT: .LBB13_2:
+; GCN-NEXT: s_endpgm
+.entry:
+ call i64 @llvm.amdgcn.struct.buffer.atomic.xor.i64(i64 1, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
+ ret void
+}
+
+
define amdgpu_cs void @atomic_xor_and_format(<4 x i32> inreg %arg) {
; IR-LABEL: define amdgpu_cs void @atomic_xor_and_format(
; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
@@ -279,14 +622,13 @@ define amdgpu_cs void @atomic_xor_and_format(<4 x i32> inreg %arg) {
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_cbranch_execz .LBB5_2
+; GCN-NEXT: s_cbranch_execz .LBB14_2
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_and_b32 s6, s6, 1
-; GCN-NEXT: s_mul_i32 s6, s6, 1
; GCN-NEXT: v_mov_b32_e32 v1, s6
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: buffer_atomic_xor v1, v2, s[0:3], 0 idxen glc
-; GCN-NEXT: .LBB5_2:
+; GCN-NEXT: .LBB14_2:
; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_readfirstlane_b32 s4, v1
@@ -332,13 +674,12 @@ define amdgpu_cs void @atomic_ptr_add(ptr addrspace(8) inreg %arg) {
; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GCN-NEXT: s_cbranch_execz .LBB6_2
+; GCN-NEXT: s_cbranch_execz .LBB15_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_mul_i32 s4, s4, 1
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: buffer_atomic_add v0, v1, s[0:3], 0 idxen
-; GCN-NEXT: .LBB6_2:
+; GCN-NEXT: .LBB15_2:
; GCN-NEXT: s_endpgm
.entry:
call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.add.i32(i32 1, ptr addrspace(8) %arg, i32 0, i32 0, i32 0, i32 0)
@@ -379,13 +720,12 @@ define amdgpu_cs void @atomic_ptr_add_and_format(ptr addrspace(8) inreg %arg) {
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_cbranch_execz .LBB7_2
+; GCN-NEXT: s_cbranch_execz .LBB16_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_mul_i32 s6, s6, 1
; GCN-NEXT: v_mov_b32_e32 v1, s6
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: buffer_atomic_add v1, v2, s[0:3], 0 idxen glc
-; GCN-NEXT: .LBB7_2:
+; GCN-NEXT: .LBB16_2:
; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_readfirstlane_b32 s4, v1
@@ -432,13 +772,12 @@ define amdgpu_cs void @atomic_ptr_sub(ptr addrspace(8) inreg %arg) {
; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GCN-NEXT: s_cbranch_execz .LBB8_2
+; GCN-NEXT: s_cbranch_execz .LBB17_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_mul_i32 s4, s4, 1
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: buffer_atomic_sub v0, v1, s[0:3], 0 idxen
-; GCN-NEXT: .LBB8_2:
+; GCN-NEXT: .LBB17_2:
; GCN-NEXT: s_endpgm
.entry:
call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.sub.i32(i32 1, ptr addrspace(8) %arg, i32 0, i32 0, i32 0, i32 0)
@@ -479,13 +818,12 @@ define amdgpu_cs void @atomic_ptr_sub_and_format(ptr addrspace(8) inreg %arg) {
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_cbranch_execz .LBB9_2
+; GCN-NEXT: s_cbranch_execz .LBB18_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_mul_i32 s6, s6, 1
; GCN-NEXT: v_mov_b32_e32 v1, s6
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: buffer_atomic_sub v1, v2, s[0:3], 0 idxen glc
-; GCN-NEXT: .LBB9_2:
+; GCN-NEXT: .LBB18_2:
; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_readfirstlane_b32 s4, v1
@@ -532,14 +870,13 @@ define amdgpu_cs void @atomic_ptr_xor(ptr addrspace(8) inreg %arg) {
; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GCN-NEXT: s_cbranch_execz .LBB10_2
+; GCN-NEXT: s_cbranch_execz .LBB19_2
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_and_b32 s4, s4, 1
-; GCN-NEXT: s_mul_i32 s4, s4, 1
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: buffer_atomic_xor v0, v1, s[0:3], 0 idxen
-; GCN-NEXT: .LBB10_2:
+; GCN-NEXT: .LBB19_2:
; GCN-NEXT: s_endpgm
.entry:
call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.xor.i32(i32 1, ptr addrspace(8) %arg, i32 0, i32 0, i32 0, i32 0)
@@ -581,14 +918,13 @@ define amdgpu_cs void @atomic_ptr_xor_and_format(ptr addrspace(8) inreg %arg) {
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_cbranch_execz .LBB11_2
+; GCN-NEXT: s_cbranch_execz .LBB20_2
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_and_b32 s6, s6, 1
-; GCN-NEXT: s_mul_i32 s6, s6, 1
; GCN-NEXT: v_mov_b32_e32 v1, s6
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: buffer_atomic_xor v1, v2, s[0:3], 0 idxen glc
-; GCN-NEXT: .LBB11_2:
+; GCN-NEXT: .LBB20_2:
; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_readfirstlane_b32 s4, v1
More information about the llvm-commits
mailing list