[llvm] [AMDGPU] Re-enable atomic optimization of uniform fadd/fsub with result (PR #97604)

Jay Foad via llvm-commits llvm-commits at lists.llvm.org
Wed Jul 3 09:25:18 PDT 2024


https://github.com/jayfoad created https://github.com/llvm/llvm-project/pull/97604

None

>From 09854ffc4d37d0a34610f5bf29cd33664e195e67 Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad at amd.com>
Date: Wed, 3 Jul 2024 17:23:19 +0100
Subject: [PATCH] [AMDGPU] Re-enable atomic optimization of uniform fadd/fsub
 with result

---
 .../Target/AMDGPU/AMDGPUAtomicOptimizer.cpp   |   21 +-
 .../global-atomicrmw-fadd-wrong-subtarget.ll  |   72 +-
 .../AMDGPU/global_atomic_optimizer_fp_rtn.ll  |  772 ++++---
 .../CodeGen/AMDGPU/local-atomicrmw-fadd.ll    | 2040 ++++++++---------
 4 files changed, 1458 insertions(+), 1447 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
index 54968be677a37..8df7d7a7a6f57 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
@@ -226,13 +226,6 @@ void AMDGPUAtomicOptimizerImpl::visitAtomicRMWInst(AtomicRMWInst &I) {
 
   bool ValDivergent = UA->isDivergentUse(I.getOperandUse(ValIdx));
 
-  if ((Op == AtomicRMWInst::FAdd || Op == AtomicRMWInst::FSub) &&
-      !I.use_empty()) {
-    // Disable the uniform return value calculation using fmul because it
-    // mishandles infinities, NaNs and signed zeros. FIXME.
-    ValDivergent = true;
-  }
-
   // If the value operand is divergent, each lane is contributing a different
   // value to the atomic calculation. We can only optimize divergent values if
   // we have DPP available on our subtarget, and the atomic operation is 32
@@ -995,13 +988,15 @@ void AMDGPUAtomicOptimizerImpl::optimizeAtomic(Instruction &I,
         break;
       case AtomicRMWInst::FAdd:
       case AtomicRMWInst::FSub: {
-        // FIXME: This path is currently disabled in visitAtomicRMWInst because
-        // of problems calculating the first active lane of the result (where
-        // Mbcnt is 0):
-        // - If V is infinity or NaN we will return NaN instead of BroadcastI.
-        // - If BroadcastI is -0.0 and V is positive we will return +0.0 instead
-        //   of -0.0.
         LaneOffset = B.CreateFMul(V, Mbcnt);
+        // The first active lane of LaneOffset needs to be the identity (-0 for
+        // fadd or +0 for fsub). The value we have calculated is V*0 which might
+        // have the wrong sign or might be nan (if V is inf or nan). Correct it
+        // with a select.
+        // TODO: We might not need this if we can prove V is not inf or nan and
+        // we don't care about signed zeros.
+        // TODO: Investigate using Intrinsic::amdgcn_fmul_legacy for this.
+        LaneOffset = B.CreateSelect(Cond, Identity, LaneOffset);
         break;
       }
       }
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd-wrong-subtarget.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd-wrong-subtarget.ll
index 32c2078f08fc0..4ac79fe9a5129 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd-wrong-subtarget.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd-wrong-subtarget.ll
@@ -4,54 +4,44 @@
 define amdgpu_kernel void @global_atomic_fadd_ret_f32_wrong_subtarget(ptr addrspace(1) %ptr) #1 {
 ; GCN-LABEL: global_atomic_fadd_ret_f32_wrong_subtarget:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    s_mov_b64 s[2:3], exec
-; GCN-NEXT:    v_bfrev_b32_e32 v1, 1
-; GCN-NEXT:    v_mov_b32_e32 v2, 4.0
-; GCN-NEXT:    ; implicit-def: $vgpr0
-; GCN-NEXT:  .LBB0_1: ; %ComputeLoop
-; GCN-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-NEXT:    s_ff1_i32_b64 s6, s[2:3]
-; GCN-NEXT:    s_lshl_b64 s[4:5], 1, s6
-; GCN-NEXT:    v_readfirstlane_b32 s7, v1
-; GCN-NEXT:    v_readlane_b32 s8, v2, s6
-; GCN-NEXT:    s_mov_b32 m0, s6
-; GCN-NEXT:    s_andn2_b64 s[2:3], s[2:3], s[4:5]
-; GCN-NEXT:    v_writelane_b32 v0, s7, m0
-; GCN-NEXT:    s_cmp_lg_u64 s[2:3], 0
-; GCN-NEXT:    v_add_f32_e32 v1, s8, v1
-; GCN-NEXT:    s_cbranch_scc1 .LBB0_1
-; GCN-NEXT:  ; %bb.2: ; %ComputeEnd
-; GCN-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GCN-NEXT:    v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
-; GCN-NEXT:    ; implicit-def: $vgpr2
+; GCN-NEXT:    s_mov_b64 s[6:7], exec
+; GCN-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
+; GCN-NEXT:    v_mbcnt_hi_u32_b32 v0, s7, v0
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT:    ; implicit-def: $vgpr1
 ; GCN-NEXT:    s_and_saveexec_b64 s[2:3], vcc
-; GCN-NEXT:    s_xor_b64 s[2:3], exec, s[2:3]
-; GCN-NEXT:    s_cbranch_execz .LBB0_6
-; GCN-NEXT:  ; %bb.3:
-; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
-; GCN-NEXT:    s_mov_b64 s[4:5], 0
-; GCN-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-NEXT:    s_cbranch_execz .LBB0_4
+; GCN-NEXT:  ; %bb.1:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GCN-NEXT:    s_bcnt1_i32_b64 s1, s[6:7]
+; GCN-NEXT:    v_cvt_f32_ubyte0_e32 v1, s1
+; GCN-NEXT:    s_mov_b64 s[6:7], 0
+; GCN-NEXT:    v_mul_f32_e32 v2, 4.0, v1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_load_dword s6, s[0:1], 0x0
+; GCN-NEXT:    s_load_dword s0, s[4:5], 0x0
+; GCN-NEXT:    v_mov_b32_e32 v3, 0
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    v_mov_b32_e32 v2, s6
-; GCN-NEXT:  .LBB0_4: ; %atomicrmw.start
+; GCN-NEXT:    v_mov_b32_e32 v1, s0
+; GCN-NEXT:  .LBB0_2: ; %atomicrmw.start
 ; GCN-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-NEXT:    v_mov_b32_e32 v5, v2
-; GCN-NEXT:    v_add_f32_e32 v4, v5, v1
-; GCN-NEXT:    global_atomic_cmpswap v2, v3, v[4:5], s[0:1] glc
+; GCN-NEXT:    v_mov_b32_e32 v5, v1
+; GCN-NEXT:    v_add_f32_e32 v4, v5, v2
+; GCN-NEXT:    global_atomic_cmpswap v1, v3, v[4:5], s[4:5] glc
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    buffer_wbinvl1
-; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v5
-; GCN-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN-NEXT:    s_cbranch_execnz .LBB0_4
-; GCN-NEXT:  ; %bb.5: ; %Flow
-; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT:  .LBB0_6: ; %Flow4
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], v1, v5
+; GCN-NEXT:    s_or_b64 s[6:7], s[0:1], s[6:7]
+; GCN-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN-NEXT:    s_cbranch_execnz .LBB0_2
+; GCN-NEXT:  ; %bb.3: ; %Flow
+; GCN-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GCN-NEXT:  .LBB0_4: ; %Flow2
 ; GCN-NEXT:    s_or_b64 exec, exec, s[2:3]
-; GCN-NEXT:    v_readfirstlane_b32 s0, v2
+; GCN-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
+; GCN-NEXT:    v_readfirstlane_b32 s0, v1
+; GCN-NEXT:    v_mul_f32_e32 v0, 4.0, v0
+; GCN-NEXT:    v_bfrev_b32_e32 v1, 1
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
 ; GCN-NEXT:    v_add_f32_e32 v0, s0, v0
 ; GCN-NEXT:    global_store_dword v[0:1], v0, off
 ; GCN-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll b/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll
index 791862dcae8dd..478ec3cfcaa4c 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll
@@ -7,101 +7,38 @@
 ; strategies are valid for only divergent values. This optimization is valid for divergent addresses. Test also covers different scopes.
 
 define amdgpu_ps float @global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, float inreg %val) #0 {
-; IR-ITERATIVE-LABEL: @global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe(
-; IR-ITERATIVE-NEXT:    [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
-; IR-ITERATIVE-NEXT:    br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP18:%.*]]
-; IR-ITERATIVE:       2:
-; IR-ITERATIVE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-ITERATIVE-NEXT:    [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
-; IR-ITERATIVE-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
-; IR-ITERATIVE-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT:    [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
-; IR-ITERATIVE-NEXT:    [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
-; IR-ITERATIVE-NEXT:    [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-ITERATIVE-NEXT:    br label [[COMPUTELOOP:%.*]]
-; IR-ITERATIVE:       10:
-; IR-ITERATIVE-NEXT:    [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP29:%.*]] syncscope("agent") monotonic, align 4
-; IR-ITERATIVE-NEXT:    br label [[TMP12:%.*]]
-; IR-ITERATIVE:       12:
-; IR-ITERATIVE-NEXT:    [[TMP13:%.*]] = phi float [ poison, [[COMPUTEEND:%.*]] ], [ [[TMP11]], [[TMP10:%.*]] ]
-; IR-ITERATIVE-NEXT:    [[TMP14:%.*]] = bitcast float [[TMP13]] to i32
-; IR-ITERATIVE-NEXT:    [[TMP15:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP14]])
-; IR-ITERATIVE-NEXT:    [[TMP16:%.*]] = bitcast i32 [[TMP15]] to float
-; IR-ITERATIVE-NEXT:    [[TMP17:%.*]] = fadd float [[TMP16]], [[TMP28:%.*]]
-; IR-ITERATIVE-NEXT:    br label [[TMP18]]
-; IR-ITERATIVE:       18:
-; IR-ITERATIVE-NEXT:    [[TMP19:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP17]], [[TMP12]] ]
-; IR-ITERATIVE-NEXT:    ret float [[TMP19]]
-; IR-ITERATIVE:       ComputeLoop:
-; IR-ITERATIVE-NEXT:    [[ACCUMULATOR:%.*]] = phi float [ -0.000000e+00, [[TMP2]] ], [ [[TMP29]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT:    [[OLDVALUEPHI:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP28]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT:    [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP32:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT:    [[TMP20:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true)
-; IR-ITERATIVE-NEXT:    [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32
-; IR-ITERATIVE-NEXT:    [[TMP22:%.*]] = bitcast float [[VAL:%.*]] to i32
-; IR-ITERATIVE-NEXT:    [[TMP23:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[TMP22]], i32 [[TMP21]])
-; IR-ITERATIVE-NEXT:    [[TMP24:%.*]] = bitcast i32 [[TMP23]] to float
-; IR-ITERATIVE-NEXT:    [[TMP25:%.*]] = bitcast float [[ACCUMULATOR]] to i32
-; IR-ITERATIVE-NEXT:    [[TMP26:%.*]] = bitcast float [[OLDVALUEPHI]] to i32
-; IR-ITERATIVE-NEXT:    [[TMP27:%.*]] = call i32 @llvm.amdgcn.writelane.i32(i32 [[TMP25]], i32 [[TMP21]], i32 [[TMP26]])
-; IR-ITERATIVE-NEXT:    [[TMP28]] = bitcast i32 [[TMP27]] to float
-; IR-ITERATIVE-NEXT:    [[TMP29]] = fadd float [[ACCUMULATOR]], [[TMP24]]
-; IR-ITERATIVE-NEXT:    [[TMP30:%.*]] = shl i64 1, [[TMP20]]
-; IR-ITERATIVE-NEXT:    [[TMP31:%.*]] = xor i64 [[TMP30]], -1
-; IR-ITERATIVE-NEXT:    [[TMP32]] = and i64 [[ACTIVEBITS]], [[TMP31]]
-; IR-ITERATIVE-NEXT:    [[TMP33:%.*]] = icmp eq i64 [[TMP32]], 0
-; IR-ITERATIVE-NEXT:    br i1 [[TMP33]], label [[COMPUTEEND]], label [[COMPUTELOOP]]
-; IR-ITERATIVE:       ComputeEnd:
-; IR-ITERATIVE-NEXT:    [[TMP34:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-ITERATIVE-NEXT:    br i1 [[TMP34]], label [[TMP10]], label [[TMP12]]
-;
-; IR-DPP-LABEL: @global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe(
-; IR-DPP-NEXT:    [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
-; IR-DPP-NEXT:    br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP40:%.*]]
-; IR-DPP:       2:
-; IR-DPP-NEXT:    [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-DPP-NEXT:    [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
-; IR-DPP-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
-; IR-DPP-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT:    [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
-; IR-DPP-NEXT:    [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
-; IR-DPP-NEXT:    [[TMP9:%.*]] = bitcast float [[VAL:%.*]] to i32
-; IR-DPP-NEXT:    [[TMP10:%.*]] = call i32 @llvm.amdgcn.set.inactive.i32(i32 [[TMP9]], i32 -2147483648)
-; IR-DPP-NEXT:    [[TMP11:%.*]] = bitcast i32 [[TMP10]] to float
-; IR-DPP-NEXT:    [[TMP12:%.*]] = bitcast i32 [[TMP9]] to float
-; IR-DPP-NEXT:    [[TMP13:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP11]], i32 273, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT:    [[TMP14:%.*]] = fadd float [[TMP11]], [[TMP13]]
-; IR-DPP-NEXT:    [[TMP15:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP14]], i32 274, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT:    [[TMP16:%.*]] = fadd float [[TMP14]], [[TMP15]]
-; IR-DPP-NEXT:    [[TMP17:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP16]], i32 276, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT:    [[TMP18:%.*]] = fadd float [[TMP16]], [[TMP17]]
-; IR-DPP-NEXT:    [[TMP19:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP18]], i32 280, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT:    [[TMP20:%.*]] = fadd float [[TMP18]], [[TMP19]]
-; IR-DPP-NEXT:    [[TMP21:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP20]], i32 322, i32 10, i32 15, i1 false)
-; IR-DPP-NEXT:    [[TMP22:%.*]] = fadd float [[TMP20]], [[TMP21]]
-; IR-DPP-NEXT:    [[TMP23:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP22]], i32 323, i32 12, i32 15, i1 false)
-; IR-DPP-NEXT:    [[TMP24:%.*]] = fadd float [[TMP22]], [[TMP23]]
-; IR-DPP-NEXT:    [[TMP25:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP24]], i32 312, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT:    [[TMP26:%.*]] = bitcast float [[TMP24]] to i32
-; IR-DPP-NEXT:    [[TMP27:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[TMP26]], i32 63)
-; IR-DPP-NEXT:    [[TMP28:%.*]] = bitcast i32 [[TMP27]] to float
-; IR-DPP-NEXT:    [[TMP29:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP28]])
-; IR-DPP-NEXT:    [[TMP30:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-DPP-NEXT:    br i1 [[TMP30]], label [[TMP31:%.*]], label [[TMP33:%.*]]
-; IR-DPP:       31:
-; IR-DPP-NEXT:    [[TMP32:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP29]] syncscope("agent") monotonic, align 4
-; IR-DPP-NEXT:    br label [[TMP33]]
-; IR-DPP:       33:
-; IR-DPP-NEXT:    [[TMP34:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP32]], [[TMP31]] ]
-; IR-DPP-NEXT:    [[TMP35:%.*]] = bitcast float [[TMP34]] to i32
-; IR-DPP-NEXT:    [[TMP36:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP35]])
-; IR-DPP-NEXT:    [[TMP37:%.*]] = bitcast i32 [[TMP36]] to float
-; IR-DPP-NEXT:    [[TMP38:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP25]])
-; IR-DPP-NEXT:    [[TMP39:%.*]] = fadd float [[TMP37]], [[TMP38]]
-; IR-DPP-NEXT:    br label [[TMP40]]
-; IR-DPP:       40:
-; IR-DPP-NEXT:    [[TMP41:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP39]], [[TMP33]] ]
-; IR-DPP-NEXT:    ret float [[TMP41]]
+; IR-LABEL: @global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe(
+; IR-NEXT:    [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
+; IR-NEXT:    br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP25:%.*]]
+; IR:       2:
+; IR-NEXT:    [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-NEXT:    [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-NEXT:    [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
+; IR-NEXT:    [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
+; IR-NEXT:    [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]])
+; IR-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
+; IR-NEXT:    [[TMP11:%.*]] = uitofp i32 [[TMP10]] to float
+; IR-NEXT:    [[TMP12:%.*]] = fmul float [[VAL:%.*]], [[TMP11]]
+; IR-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
+; IR:       14:
+; IR-NEXT:    [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] syncscope("agent") monotonic, align 4
+; IR-NEXT:    br label [[TMP16]]
+; IR:       16:
+; IR-NEXT:    [[TMP17:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
+; IR-NEXT:    [[TMP18:%.*]] = bitcast float [[TMP17]] to i32
+; IR-NEXT:    [[TMP19:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP18]])
+; IR-NEXT:    [[TMP20:%.*]] = bitcast i32 [[TMP19]] to float
+; IR-NEXT:    [[TMP21:%.*]] = uitofp i32 [[TMP8]] to float
+; IR-NEXT:    [[TMP22:%.*]] = fmul float [[VAL]], [[TMP21]]
+; IR-NEXT:    [[TMP23:%.*]] = select i1 [[TMP13]], float -0.000000e+00, float [[TMP22]]
+; IR-NEXT:    [[TMP24:%.*]] = fadd float [[TMP20]], [[TMP23]]
+; IR-NEXT:    br label [[TMP25]]
+; IR:       25:
+; IR-NEXT:    [[TMP26:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP24]], [[TMP16]] ]
+; IR-NEXT:    ret float [[TMP26]]
 ;
   %result = atomicrmw fadd ptr addrspace(1) %ptr, float %val syncscope("agent") monotonic, align 4
   ret float %result
@@ -211,7 +148,7 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_div_value_scope_agent_sco
 define amdgpu_ps float @global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, float inreg %val) #1 {
 ; IR-ITERATIVE-LABEL: @global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_strictfp(
 ; IR-ITERATIVE-NEXT:    [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7:[0-9]+]]
-; IR-ITERATIVE-NEXT:    br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP18:%.*]]
+; IR-ITERATIVE-NEXT:    br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP25:%.*]]
 ; IR-ITERATIVE:       2:
 ; IR-ITERATIVE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
 ; IR-ITERATIVE-NEXT:    [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
@@ -219,47 +156,32 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_uni_value_one_as_scope_un
 ; IR-ITERATIVE-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
 ; IR-ITERATIVE-NEXT:    [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
 ; IR-ITERATIVE-NEXT:    [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT:    [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT:    br label [[COMPUTELOOP:%.*]]
-; IR-ITERATIVE:       10:
-; IR-ITERATIVE-NEXT:    [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP29:%.*]] syncscope("one-as") monotonic, align 4
-; IR-ITERATIVE-NEXT:    br label [[TMP12:%.*]]
-; IR-ITERATIVE:       12:
-; IR-ITERATIVE-NEXT:    [[TMP13:%.*]] = phi float [ poison, [[COMPUTEEND:%.*]] ], [ [[TMP11]], [[TMP10:%.*]] ]
-; IR-ITERATIVE-NEXT:    [[TMP14:%.*]] = bitcast float [[TMP13]] to i32
-; IR-ITERATIVE-NEXT:    [[TMP15:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP14]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT:    [[TMP16:%.*]] = bitcast i32 [[TMP15]] to float
-; IR-ITERATIVE-NEXT:    [[TMP17:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP16]], float [[TMP28:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT:    br label [[TMP18]]
-; IR-ITERATIVE:       18:
-; IR-ITERATIVE-NEXT:    [[TMP19:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP17]], [[TMP12]] ]
-; IR-ITERATIVE-NEXT:    ret float [[TMP19]]
-; IR-ITERATIVE:       ComputeLoop:
-; IR-ITERATIVE-NEXT:    [[ACCUMULATOR:%.*]] = phi float [ -0.000000e+00, [[TMP2]] ], [ [[TMP29]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT:    [[OLDVALUEPHI:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP28]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT:    [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP32:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT:    [[TMP20:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT:    [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32
-; IR-ITERATIVE-NEXT:    [[TMP22:%.*]] = bitcast float [[VAL:%.*]] to i32
-; IR-ITERATIVE-NEXT:    [[TMP23:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[TMP22]], i32 [[TMP21]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT:    [[TMP24:%.*]] = bitcast i32 [[TMP23]] to float
-; IR-ITERATIVE-NEXT:    [[TMP25:%.*]] = bitcast float [[ACCUMULATOR]] to i32
-; IR-ITERATIVE-NEXT:    [[TMP26:%.*]] = bitcast float [[OLDVALUEPHI]] to i32
-; IR-ITERATIVE-NEXT:    [[TMP27:%.*]] = call i32 @llvm.amdgcn.writelane.i32(i32 [[TMP25]], i32 [[TMP21]], i32 [[TMP26]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT:    [[TMP28]] = bitcast i32 [[TMP27]] to float
-; IR-ITERATIVE-NEXT:    [[TMP29]] = call float @llvm.experimental.constrained.fadd.f32(float [[ACCUMULATOR]], float [[TMP24]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT:    [[TMP30:%.*]] = shl i64 1, [[TMP20]]
-; IR-ITERATIVE-NEXT:    [[TMP31:%.*]] = xor i64 [[TMP30]], -1
-; IR-ITERATIVE-NEXT:    [[TMP32]] = and i64 [[ACTIVEBITS]], [[TMP31]]
-; IR-ITERATIVE-NEXT:    [[TMP33:%.*]] = icmp eq i64 [[TMP32]], 0
-; IR-ITERATIVE-NEXT:    br i1 [[TMP33]], label [[COMPUTEEND]], label [[COMPUTELOOP]]
-; IR-ITERATIVE:       ComputeEnd:
-; IR-ITERATIVE-NEXT:    [[TMP34:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-ITERATIVE-NEXT:    br i1 [[TMP34]], label [[TMP10]], label [[TMP12]]
+; IR-ITERATIVE-NEXT:    [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
+; IR-ITERATIVE-NEXT:    [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
+; IR-ITERATIVE:       14:
+; IR-ITERATIVE-NEXT:    [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] syncscope("one-as") monotonic, align 4
+; IR-ITERATIVE-NEXT:    br label [[TMP16]]
+; IR-ITERATIVE:       16:
+; IR-ITERATIVE-NEXT:    [[TMP17:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
+; IR-ITERATIVE-NEXT:    [[TMP18:%.*]] = bitcast float [[TMP17]] to i32
+; IR-ITERATIVE-NEXT:    [[TMP19:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP18]]) #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP20:%.*]] = bitcast i32 [[TMP19]] to float
+; IR-ITERATIVE-NEXT:    [[TMP21:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP22:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP23:%.*]] = select i1 [[TMP13]], float -0.000000e+00, float [[TMP22]]
+; IR-ITERATIVE-NEXT:    [[TMP24:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP20]], float [[TMP23]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    br label [[TMP25]]
+; IR-ITERATIVE:       25:
+; IR-ITERATIVE-NEXT:    [[TMP26:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP24]], [[TMP16]] ]
+; IR-ITERATIVE-NEXT:    ret float [[TMP26]]
 ;
 ; IR-DPP-LABEL: @global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_strictfp(
 ; IR-DPP-NEXT:    [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8:[0-9]+]]
-; IR-DPP-NEXT:    br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP40:%.*]]
+; IR-DPP-NEXT:    br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP25:%.*]]
 ; IR-DPP:       2:
 ; IR-DPP-NEXT:    [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
 ; IR-DPP-NEXT:    [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
@@ -267,43 +189,28 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_uni_value_one_as_scope_un
 ; IR-DPP-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
 ; IR-DPP-NEXT:    [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
 ; IR-DPP-NEXT:    [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP9:%.*]] = bitcast float [[VAL:%.*]] to i32
-; IR-DPP-NEXT:    [[TMP10:%.*]] = call i32 @llvm.amdgcn.set.inactive.i32(i32 [[TMP9]], i32 -2147483648) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP11:%.*]] = bitcast i32 [[TMP10]] to float
-; IR-DPP-NEXT:    [[TMP12:%.*]] = bitcast i32 [[TMP9]] to float
-; IR-DPP-NEXT:    [[TMP13:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP11]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP14:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP11]], float [[TMP13]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP15:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP14]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP16:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP14]], float [[TMP15]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP17:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP16]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP18:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP16]], float [[TMP17]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP19:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP18]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP20:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP18]], float [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP21:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP20]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP22:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP20]], float [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP23:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP22]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP24:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP22]], float [[TMP23]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP25:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP24]], i32 312, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP26:%.*]] = bitcast float [[TMP24]] to i32
-; IR-DPP-NEXT:    [[TMP27:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[TMP26]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP28:%.*]] = bitcast i32 [[TMP27]] to float
-; IR-DPP-NEXT:    [[TMP29:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP28]]) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP30:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-DPP-NEXT:    br i1 [[TMP30]], label [[TMP31:%.*]], label [[TMP33:%.*]]
-; IR-DPP:       31:
-; IR-DPP-NEXT:    [[TMP32:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP29]] syncscope("one-as") monotonic, align 4
-; IR-DPP-NEXT:    br label [[TMP33]]
-; IR-DPP:       33:
-; IR-DPP-NEXT:    [[TMP34:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP32]], [[TMP31]] ]
-; IR-DPP-NEXT:    [[TMP35:%.*]] = bitcast float [[TMP34]] to i32
-; IR-DPP-NEXT:    [[TMP36:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP35]]) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP37:%.*]] = bitcast i32 [[TMP36]] to float
-; IR-DPP-NEXT:    [[TMP38:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP25]]) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP39:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP37]], float [[TMP38]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT:    br label [[TMP40]]
-; IR-DPP:       40:
-; IR-DPP-NEXT:    [[TMP41:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP39]], [[TMP33]] ]
-; IR-DPP-NEXT:    ret float [[TMP41]]
+; IR-DPP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
+; IR-DPP-NEXT:    [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-DPP-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
+; IR-DPP:       14:
+; IR-DPP-NEXT:    [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] syncscope("one-as") monotonic, align 4
+; IR-DPP-NEXT:    br label [[TMP16]]
+; IR-DPP:       16:
+; IR-DPP-NEXT:    [[TMP17:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
+; IR-DPP-NEXT:    [[TMP18:%.*]] = bitcast float [[TMP17]] to i32
+; IR-DPP-NEXT:    [[TMP19:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP18]]) #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP20:%.*]] = bitcast i32 [[TMP19]] to float
+; IR-DPP-NEXT:    [[TMP21:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP22:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP23:%.*]] = select i1 [[TMP13]], float -0.000000e+00, float [[TMP22]]
+; IR-DPP-NEXT:    [[TMP24:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP20]], float [[TMP23]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT:    br label [[TMP25]]
+; IR-DPP:       25:
+; IR-DPP-NEXT:    [[TMP26:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP24]], [[TMP16]] ]
+; IR-DPP-NEXT:    ret float [[TMP26]]
 ;
   %result = atomicrmw fadd ptr addrspace(1) %ptr, float %val syncscope("one-as") monotonic
   ret float %result
@@ -413,7 +320,7 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_div_value_one_as_scope_un
 define amdgpu_ps float @global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp(ptr addrspace(1) inreg %ptr, float inreg %val) #2 {
 ; IR-ITERATIVE-LABEL: @global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp(
 ; IR-ITERATIVE-NEXT:    [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
-; IR-ITERATIVE-NEXT:    br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP18:%.*]]
+; IR-ITERATIVE-NEXT:    br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP25:%.*]]
 ; IR-ITERATIVE:       2:
 ; IR-ITERATIVE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
 ; IR-ITERATIVE-NEXT:    [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
@@ -421,47 +328,32 @@ define amdgpu_ps float @global_atomic_fsub_uni_address_uni_value_agent_scope_str
 ; IR-ITERATIVE-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
 ; IR-ITERATIVE-NEXT:    [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
 ; IR-ITERATIVE-NEXT:    [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT:    [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT:    br label [[COMPUTELOOP:%.*]]
-; IR-ITERATIVE:       10:
-; IR-ITERATIVE-NEXT:    [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP29:%.*]] syncscope("agent") monotonic, align 4
-; IR-ITERATIVE-NEXT:    br label [[TMP12:%.*]]
-; IR-ITERATIVE:       12:
-; IR-ITERATIVE-NEXT:    [[TMP13:%.*]] = phi float [ poison, [[COMPUTEEND:%.*]] ], [ [[TMP11]], [[TMP10:%.*]] ]
-; IR-ITERATIVE-NEXT:    [[TMP14:%.*]] = bitcast float [[TMP13]] to i32
-; IR-ITERATIVE-NEXT:    [[TMP15:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP14]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT:    [[TMP16:%.*]] = bitcast i32 [[TMP15]] to float
-; IR-ITERATIVE-NEXT:    [[TMP17:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP16]], float [[TMP28:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT:    br label [[TMP18]]
-; IR-ITERATIVE:       18:
-; IR-ITERATIVE-NEXT:    [[TMP19:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP17]], [[TMP12]] ]
-; IR-ITERATIVE-NEXT:    ret float [[TMP19]]
-; IR-ITERATIVE:       ComputeLoop:
-; IR-ITERATIVE-NEXT:    [[ACCUMULATOR:%.*]] = phi float [ -0.000000e+00, [[TMP2]] ], [ [[TMP29]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT:    [[OLDVALUEPHI:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP28]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT:    [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP32:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT:    [[TMP20:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT:    [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32
-; IR-ITERATIVE-NEXT:    [[TMP22:%.*]] = bitcast float [[VAL:%.*]] to i32
-; IR-ITERATIVE-NEXT:    [[TMP23:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[TMP22]], i32 [[TMP21]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT:    [[TMP24:%.*]] = bitcast i32 [[TMP23]] to float
-; IR-ITERATIVE-NEXT:    [[TMP25:%.*]] = bitcast float [[ACCUMULATOR]] to i32
-; IR-ITERATIVE-NEXT:    [[TMP26:%.*]] = bitcast float [[OLDVALUEPHI]] to i32
-; IR-ITERATIVE-NEXT:    [[TMP27:%.*]] = call i32 @llvm.amdgcn.writelane.i32(i32 [[TMP25]], i32 [[TMP21]], i32 [[TMP26]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT:    [[TMP28]] = bitcast i32 [[TMP27]] to float
-; IR-ITERATIVE-NEXT:    [[TMP29]] = call float @llvm.experimental.constrained.fadd.f32(float [[ACCUMULATOR]], float [[TMP24]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT:    [[TMP30:%.*]] = shl i64 1, [[TMP20]]
-; IR-ITERATIVE-NEXT:    [[TMP31:%.*]] = xor i64 [[TMP30]], -1
-; IR-ITERATIVE-NEXT:    [[TMP32]] = and i64 [[ACTIVEBITS]], [[TMP31]]
-; IR-ITERATIVE-NEXT:    [[TMP33:%.*]] = icmp eq i64 [[TMP32]], 0
-; IR-ITERATIVE-NEXT:    br i1 [[TMP33]], label [[COMPUTEEND]], label [[COMPUTELOOP]]
-; IR-ITERATIVE:       ComputeEnd:
-; IR-ITERATIVE-NEXT:    [[TMP34:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-ITERATIVE-NEXT:    br i1 [[TMP34]], label [[TMP10]], label [[TMP12]]
+; IR-ITERATIVE-NEXT:    [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
+; IR-ITERATIVE-NEXT:    [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
+; IR-ITERATIVE:       14:
+; IR-ITERATIVE-NEXT:    [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] syncscope("agent") monotonic, align 4
+; IR-ITERATIVE-NEXT:    br label [[TMP16]]
+; IR-ITERATIVE:       16:
+; IR-ITERATIVE-NEXT:    [[TMP17:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
+; IR-ITERATIVE-NEXT:    [[TMP18:%.*]] = bitcast float [[TMP17]] to i32
+; IR-ITERATIVE-NEXT:    [[TMP19:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP18]]) #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP20:%.*]] = bitcast i32 [[TMP19]] to float
+; IR-ITERATIVE-NEXT:    [[TMP21:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP22:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP23:%.*]] = select i1 [[TMP13]], float -0.000000e+00, float [[TMP22]]
+; IR-ITERATIVE-NEXT:    [[TMP24:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP20]], float [[TMP23]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    br label [[TMP25]]
+; IR-ITERATIVE:       25:
+; IR-ITERATIVE-NEXT:    [[TMP26:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP24]], [[TMP16]] ]
+; IR-ITERATIVE-NEXT:    ret float [[TMP26]]
 ;
 ; IR-DPP-LABEL: @global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp(
 ; IR-DPP-NEXT:    [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
-; IR-DPP-NEXT:    br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP40:%.*]]
+; IR-DPP-NEXT:    br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP25:%.*]]
 ; IR-DPP:       2:
 ; IR-DPP-NEXT:    [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
 ; IR-DPP-NEXT:    [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
@@ -469,43 +361,28 @@ define amdgpu_ps float @global_atomic_fsub_uni_address_uni_value_agent_scope_str
 ; IR-DPP-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
 ; IR-DPP-NEXT:    [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
 ; IR-DPP-NEXT:    [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP9:%.*]] = bitcast float [[VAL:%.*]] to i32
-; IR-DPP-NEXT:    [[TMP10:%.*]] = call i32 @llvm.amdgcn.set.inactive.i32(i32 [[TMP9]], i32 -2147483648) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP11:%.*]] = bitcast i32 [[TMP10]] to float
-; IR-DPP-NEXT:    [[TMP12:%.*]] = bitcast i32 [[TMP9]] to float
-; IR-DPP-NEXT:    [[TMP13:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP11]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP14:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP11]], float [[TMP13]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP15:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP14]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP16:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP14]], float [[TMP15]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP17:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP16]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP18:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP16]], float [[TMP17]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP19:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP18]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP20:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP18]], float [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP21:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP20]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP22:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP20]], float [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP23:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP22]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP24:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP22]], float [[TMP23]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP25:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP24]], i32 312, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP26:%.*]] = bitcast float [[TMP24]] to i32
-; IR-DPP-NEXT:    [[TMP27:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[TMP26]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP28:%.*]] = bitcast i32 [[TMP27]] to float
-; IR-DPP-NEXT:    [[TMP29:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP28]]) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP30:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-DPP-NEXT:    br i1 [[TMP30]], label [[TMP31:%.*]], label [[TMP33:%.*]]
-; IR-DPP:       31:
-; IR-DPP-NEXT:    [[TMP32:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP29]] syncscope("agent") monotonic, align 4
-; IR-DPP-NEXT:    br label [[TMP33]]
-; IR-DPP:       33:
-; IR-DPP-NEXT:    [[TMP34:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP32]], [[TMP31]] ]
-; IR-DPP-NEXT:    [[TMP35:%.*]] = bitcast float [[TMP34]] to i32
-; IR-DPP-NEXT:    [[TMP36:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP35]]) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP37:%.*]] = bitcast i32 [[TMP36]] to float
-; IR-DPP-NEXT:    [[TMP38:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP25]]) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP39:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP37]], float [[TMP38]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT:    br label [[TMP40]]
-; IR-DPP:       40:
-; IR-DPP-NEXT:    [[TMP41:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP39]], [[TMP33]] ]
-; IR-DPP-NEXT:    ret float [[TMP41]]
+; IR-DPP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
+; IR-DPP-NEXT:    [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-DPP-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
+; IR-DPP:       14:
+; IR-DPP-NEXT:    [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] syncscope("agent") monotonic, align 4
+; IR-DPP-NEXT:    br label [[TMP16]]
+; IR-DPP:       16:
+; IR-DPP-NEXT:    [[TMP17:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
+; IR-DPP-NEXT:    [[TMP18:%.*]] = bitcast float [[TMP17]] to i32
+; IR-DPP-NEXT:    [[TMP19:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP18]]) #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP20:%.*]] = bitcast i32 [[TMP19]] to float
+; IR-DPP-NEXT:    [[TMP21:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP22:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP23:%.*]] = select i1 [[TMP13]], float -0.000000e+00, float [[TMP22]]
+; IR-DPP-NEXT:    [[TMP24:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP20]], float [[TMP23]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT:    br label [[TMP25]]
+; IR-DPP:       25:
+; IR-DPP-NEXT:    [[TMP26:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP24]], [[TMP16]] ]
+; IR-DPP-NEXT:    ret float [[TMP26]]
 ;
   %result = atomicrmw fadd ptr addrspace(1) %ptr, float %val syncscope("agent") monotonic
   ret float %result
@@ -911,7 +788,7 @@ define amdgpu_ps float @global_atomic_fmax_uni_address_div_value_agent_scope_uns
 define amdgpu_ps float @global_atomic_fadd_uni_address_uni_value_system_scope_strictfp(ptr addrspace(1) inreg %ptr, float inreg %val) #2 {
 ; IR-ITERATIVE-LABEL: @global_atomic_fadd_uni_address_uni_value_system_scope_strictfp(
 ; IR-ITERATIVE-NEXT:    [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
-; IR-ITERATIVE-NEXT:    br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP18:%.*]]
+; IR-ITERATIVE-NEXT:    br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP25:%.*]]
 ; IR-ITERATIVE:       2:
 ; IR-ITERATIVE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
 ; IR-ITERATIVE-NEXT:    [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
@@ -919,47 +796,32 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_uni_value_system_scope_st
 ; IR-ITERATIVE-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
 ; IR-ITERATIVE-NEXT:    [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
 ; IR-ITERATIVE-NEXT:    [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT:    [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT:    br label [[COMPUTELOOP:%.*]]
-; IR-ITERATIVE:       10:
-; IR-ITERATIVE-NEXT:    [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP29:%.*]] monotonic, align 4
-; IR-ITERATIVE-NEXT:    br label [[TMP12:%.*]]
-; IR-ITERATIVE:       12:
-; IR-ITERATIVE-NEXT:    [[TMP13:%.*]] = phi float [ poison, [[COMPUTEEND:%.*]] ], [ [[TMP11]], [[TMP10:%.*]] ]
-; IR-ITERATIVE-NEXT:    [[TMP14:%.*]] = bitcast float [[TMP13]] to i32
-; IR-ITERATIVE-NEXT:    [[TMP15:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP14]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT:    [[TMP16:%.*]] = bitcast i32 [[TMP15]] to float
-; IR-ITERATIVE-NEXT:    [[TMP17:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP16]], float [[TMP28:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT:    br label [[TMP18]]
-; IR-ITERATIVE:       18:
-; IR-ITERATIVE-NEXT:    [[TMP19:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP17]], [[TMP12]] ]
-; IR-ITERATIVE-NEXT:    ret float [[TMP19]]
-; IR-ITERATIVE:       ComputeLoop:
-; IR-ITERATIVE-NEXT:    [[ACCUMULATOR:%.*]] = phi float [ -0.000000e+00, [[TMP2]] ], [ [[TMP29]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT:    [[OLDVALUEPHI:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP28]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT:    [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP32:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT:    [[TMP20:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT:    [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32
-; IR-ITERATIVE-NEXT:    [[TMP22:%.*]] = bitcast float [[VAL:%.*]] to i32
-; IR-ITERATIVE-NEXT:    [[TMP23:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[TMP22]], i32 [[TMP21]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT:    [[TMP24:%.*]] = bitcast i32 [[TMP23]] to float
-; IR-ITERATIVE-NEXT:    [[TMP25:%.*]] = bitcast float [[ACCUMULATOR]] to i32
-; IR-ITERATIVE-NEXT:    [[TMP26:%.*]] = bitcast float [[OLDVALUEPHI]] to i32
-; IR-ITERATIVE-NEXT:    [[TMP27:%.*]] = call i32 @llvm.amdgcn.writelane.i32(i32 [[TMP25]], i32 [[TMP21]], i32 [[TMP26]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT:    [[TMP28]] = bitcast i32 [[TMP27]] to float
-; IR-ITERATIVE-NEXT:    [[TMP29]] = call float @llvm.experimental.constrained.fadd.f32(float [[ACCUMULATOR]], float [[TMP24]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT:    [[TMP30:%.*]] = shl i64 1, [[TMP20]]
-; IR-ITERATIVE-NEXT:    [[TMP31:%.*]] = xor i64 [[TMP30]], -1
-; IR-ITERATIVE-NEXT:    [[TMP32]] = and i64 [[ACTIVEBITS]], [[TMP31]]
-; IR-ITERATIVE-NEXT:    [[TMP33:%.*]] = icmp eq i64 [[TMP32]], 0
-; IR-ITERATIVE-NEXT:    br i1 [[TMP33]], label [[COMPUTEEND]], label [[COMPUTELOOP]]
-; IR-ITERATIVE:       ComputeEnd:
-; IR-ITERATIVE-NEXT:    [[TMP34:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-ITERATIVE-NEXT:    br i1 [[TMP34]], label [[TMP10]], label [[TMP12]]
+; IR-ITERATIVE-NEXT:    [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
+; IR-ITERATIVE-NEXT:    [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
+; IR-ITERATIVE:       14:
+; IR-ITERATIVE-NEXT:    [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] monotonic, align 4
+; IR-ITERATIVE-NEXT:    br label [[TMP16]]
+; IR-ITERATIVE:       16:
+; IR-ITERATIVE-NEXT:    [[TMP17:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
+; IR-ITERATIVE-NEXT:    [[TMP18:%.*]] = bitcast float [[TMP17]] to i32
+; IR-ITERATIVE-NEXT:    [[TMP19:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP18]]) #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP20:%.*]] = bitcast i32 [[TMP19]] to float
+; IR-ITERATIVE-NEXT:    [[TMP21:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP22:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP23:%.*]] = select i1 [[TMP13]], float -0.000000e+00, float [[TMP22]]
+; IR-ITERATIVE-NEXT:    [[TMP24:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP20]], float [[TMP23]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    br label [[TMP25]]
+; IR-ITERATIVE:       25:
+; IR-ITERATIVE-NEXT:    [[TMP26:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP24]], [[TMP16]] ]
+; IR-ITERATIVE-NEXT:    ret float [[TMP26]]
 ;
 ; IR-DPP-LABEL: @global_atomic_fadd_uni_address_uni_value_system_scope_strictfp(
 ; IR-DPP-NEXT:    [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
-; IR-DPP-NEXT:    br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP40:%.*]]
+; IR-DPP-NEXT:    br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP25:%.*]]
 ; IR-DPP:       2:
 ; IR-DPP-NEXT:    [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
 ; IR-DPP-NEXT:    [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
@@ -967,43 +829,28 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_uni_value_system_scope_st
 ; IR-DPP-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
 ; IR-DPP-NEXT:    [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
 ; IR-DPP-NEXT:    [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP9:%.*]] = bitcast float [[VAL:%.*]] to i32
-; IR-DPP-NEXT:    [[TMP10:%.*]] = call i32 @llvm.amdgcn.set.inactive.i32(i32 [[TMP9]], i32 -2147483648) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP11:%.*]] = bitcast i32 [[TMP10]] to float
-; IR-DPP-NEXT:    [[TMP12:%.*]] = bitcast i32 [[TMP9]] to float
-; IR-DPP-NEXT:    [[TMP13:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP11]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP14:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP11]], float [[TMP13]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP15:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP14]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP16:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP14]], float [[TMP15]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP17:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP16]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP18:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP16]], float [[TMP17]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP19:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP18]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP20:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP18]], float [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP21:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP20]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP22:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP20]], float [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP23:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP22]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP24:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP22]], float [[TMP23]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP25:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP24]], i32 312, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP26:%.*]] = bitcast float [[TMP24]] to i32
-; IR-DPP-NEXT:    [[TMP27:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[TMP26]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP28:%.*]] = bitcast i32 [[TMP27]] to float
-; IR-DPP-NEXT:    [[TMP29:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP28]]) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP30:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-DPP-NEXT:    br i1 [[TMP30]], label [[TMP31:%.*]], label [[TMP33:%.*]]
-; IR-DPP:       31:
-; IR-DPP-NEXT:    [[TMP32:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP29]] monotonic, align 4
-; IR-DPP-NEXT:    br label [[TMP33]]
-; IR-DPP:       33:
-; IR-DPP-NEXT:    [[TMP34:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP32]], [[TMP31]] ]
-; IR-DPP-NEXT:    [[TMP35:%.*]] = bitcast float [[TMP34]] to i32
-; IR-DPP-NEXT:    [[TMP36:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP35]]) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP37:%.*]] = bitcast i32 [[TMP36]] to float
-; IR-DPP-NEXT:    [[TMP38:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP25]]) #[[ATTR8]]
-; IR-DPP-NEXT:    [[TMP39:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP37]], float [[TMP38]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT:    br label [[TMP40]]
-; IR-DPP:       40:
-; IR-DPP-NEXT:    [[TMP41:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP39]], [[TMP33]] ]
-; IR-DPP-NEXT:    ret float [[TMP41]]
+; IR-DPP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
+; IR-DPP-NEXT:    [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-DPP-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
+; IR-DPP:       14:
+; IR-DPP-NEXT:    [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] monotonic, align 4
+; IR-DPP-NEXT:    br label [[TMP16]]
+; IR-DPP:       16:
+; IR-DPP-NEXT:    [[TMP17:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
+; IR-DPP-NEXT:    [[TMP18:%.*]] = bitcast float [[TMP17]] to i32
+; IR-DPP-NEXT:    [[TMP19:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP18]]) #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP20:%.*]] = bitcast i32 [[TMP19]] to float
+; IR-DPP-NEXT:    [[TMP21:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP22:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP23:%.*]] = select i1 [[TMP13]], float -0.000000e+00, float [[TMP22]]
+; IR-DPP-NEXT:    [[TMP24:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP20]], float [[TMP23]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT:    br label [[TMP25]]
+; IR-DPP:       25:
+; IR-DPP-NEXT:    [[TMP26:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP24]], [[TMP16]] ]
+; IR-DPP-NEXT:    ret float [[TMP26]]
 ;
   %result = atomicrmw fadd ptr addrspace(1) %ptr, float %val monotonic, align 4
   ret float %result
@@ -1220,8 +1067,43 @@ define amdgpu_ps float @global_atomic_fadd_div_address_div_value_system_scope_st
 
 define amdgpu_ps double @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double inreg %val) #0 {
 ; IR-LABEL: @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe(
-; IR-NEXT:    [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 4
-; IR-NEXT:    ret double [[RESULT]]
+; IR-NEXT:    [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
+; IR-NEXT:    br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP31:%.*]]
+; IR:       2:
+; IR-NEXT:    [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-NEXT:    [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-NEXT:    [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
+; IR-NEXT:    [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
+; IR-NEXT:    [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]])
+; IR-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
+; IR-NEXT:    [[TMP11:%.*]] = uitofp i32 [[TMP10]] to double
+; IR-NEXT:    [[TMP12:%.*]] = fmul double [[VAL:%.*]], [[TMP11]]
+; IR-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
+; IR:       14:
+; IR-NEXT:    [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 4
+; IR-NEXT:    br label [[TMP16]]
+; IR:       16:
+; IR-NEXT:    [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
+; IR-NEXT:    [[TMP18:%.*]] = bitcast double [[TMP17]] to i64
+; IR-NEXT:    [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32
+; IR-NEXT:    [[TMP20:%.*]] = lshr i64 [[TMP18]], 32
+; IR-NEXT:    [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32
+; IR-NEXT:    [[TMP22:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP19]])
+; IR-NEXT:    [[TMP23:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP21]])
+; IR-NEXT:    [[TMP24:%.*]] = insertelement <2 x i32> poison, i32 [[TMP22]], i32 0
+; IR-NEXT:    [[TMP25:%.*]] = insertelement <2 x i32> [[TMP24]], i32 [[TMP23]], i32 1
+; IR-NEXT:    [[TMP26:%.*]] = bitcast <2 x i32> [[TMP25]] to double
+; IR-NEXT:    [[TMP27:%.*]] = uitofp i32 [[TMP8]] to double
+; IR-NEXT:    [[TMP28:%.*]] = fmul double [[VAL]], [[TMP27]]
+; IR-NEXT:    [[TMP29:%.*]] = select i1 [[TMP13]], double -0.000000e+00, double [[TMP28]]
+; IR-NEXT:    [[TMP30:%.*]] = fadd double [[TMP26]], [[TMP29]]
+; IR-NEXT:    br label [[TMP31]]
+; IR:       31:
+; IR-NEXT:    [[TMP32:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP30]], [[TMP16]] ]
+; IR-NEXT:    ret double [[TMP32]]
 ;
   %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic, align 4
   ret double %result
@@ -1237,9 +1119,83 @@ define amdgpu_ps double @global_atomic_fadd_double_uni_address_div_value_scope_a
 }
 
 define amdgpu_ps double @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #1 {
-; IR-LABEL: @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_strictfp(
-; IR-NEXT:    [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("one-as") monotonic, align 8
-; IR-NEXT:    ret double [[RESULT]]
+; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_strictfp(
+; IR-ITERATIVE-NEXT:    [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP31:%.*]]
+; IR-ITERATIVE:       2:
+; IR-ITERATIVE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-ITERATIVE-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-ITERATIVE-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-ITERATIVE-NEXT:    [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
+; IR-ITERATIVE-NEXT:    [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
+; IR-ITERATIVE:       14:
+; IR-ITERATIVE-NEXT:    [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("one-as") monotonic, align 8
+; IR-ITERATIVE-NEXT:    br label [[TMP16]]
+; IR-ITERATIVE:       16:
+; IR-ITERATIVE-NEXT:    [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
+; IR-ITERATIVE-NEXT:    [[TMP18:%.*]] = bitcast double [[TMP17]] to i64
+; IR-ITERATIVE-NEXT:    [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32
+; IR-ITERATIVE-NEXT:    [[TMP20:%.*]] = lshr i64 [[TMP18]], 32
+; IR-ITERATIVE-NEXT:    [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32
+; IR-ITERATIVE-NEXT:    [[TMP22:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP19]]) #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP23:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP21]]) #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP24:%.*]] = insertelement <2 x i32> poison, i32 [[TMP22]], i32 0
+; IR-ITERATIVE-NEXT:    [[TMP25:%.*]] = insertelement <2 x i32> [[TMP24]], i32 [[TMP23]], i32 1
+; IR-ITERATIVE-NEXT:    [[TMP26:%.*]] = bitcast <2 x i32> [[TMP25]] to double
+; IR-ITERATIVE-NEXT:    [[TMP27:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP28:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP27]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP29:%.*]] = select i1 [[TMP13]], double -0.000000e+00, double [[TMP28]]
+; IR-ITERATIVE-NEXT:    [[TMP30:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP26]], double [[TMP29]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    br label [[TMP31]]
+; IR-ITERATIVE:       31:
+; IR-ITERATIVE-NEXT:    [[TMP32:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP30]], [[TMP16]] ]
+; IR-ITERATIVE-NEXT:    ret double [[TMP32]]
+;
+; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_strictfp(
+; IR-DPP-NEXT:    [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT:    br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP31:%.*]]
+; IR-DPP:       2:
+; IR-DPP-NEXT:    [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-DPP-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-DPP-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-DPP-NEXT:    [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
+; IR-DPP-NEXT:    [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-DPP-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
+; IR-DPP:       14:
+; IR-DPP-NEXT:    [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("one-as") monotonic, align 8
+; IR-DPP-NEXT:    br label [[TMP16]]
+; IR-DPP:       16:
+; IR-DPP-NEXT:    [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
+; IR-DPP-NEXT:    [[TMP18:%.*]] = bitcast double [[TMP17]] to i64
+; IR-DPP-NEXT:    [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32
+; IR-DPP-NEXT:    [[TMP20:%.*]] = lshr i64 [[TMP18]], 32
+; IR-DPP-NEXT:    [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32
+; IR-DPP-NEXT:    [[TMP22:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP19]]) #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP23:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP21]]) #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP24:%.*]] = insertelement <2 x i32> poison, i32 [[TMP22]], i32 0
+; IR-DPP-NEXT:    [[TMP25:%.*]] = insertelement <2 x i32> [[TMP24]], i32 [[TMP23]], i32 1
+; IR-DPP-NEXT:    [[TMP26:%.*]] = bitcast <2 x i32> [[TMP25]] to double
+; IR-DPP-NEXT:    [[TMP27:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP28:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP27]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP29:%.*]] = select i1 [[TMP13]], double -0.000000e+00, double [[TMP28]]
+; IR-DPP-NEXT:    [[TMP30:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP26]], double [[TMP29]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT:    br label [[TMP31]]
+; IR-DPP:       31:
+; IR-DPP-NEXT:    [[TMP32:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP30]], [[TMP16]] ]
+; IR-DPP-NEXT:    ret double [[TMP32]]
 ;
   %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("one-as") monotonic
   ret double %result
@@ -1255,9 +1211,83 @@ define amdgpu_ps double @global_atomic_fadd_double_uni_address_div_value_one_as_
 }
 
 define amdgpu_ps double @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #2 {
-; IR-LABEL: @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp(
-; IR-NEXT:    [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8
-; IR-NEXT:    ret double [[RESULT]]
+; IR-ITERATIVE-LABEL: @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp(
+; IR-ITERATIVE-NEXT:    [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP31:%.*]]
+; IR-ITERATIVE:       2:
+; IR-ITERATIVE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-ITERATIVE-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-ITERATIVE-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-ITERATIVE-NEXT:    [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
+; IR-ITERATIVE-NEXT:    [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
+; IR-ITERATIVE:       14:
+; IR-ITERATIVE-NEXT:    [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 8
+; IR-ITERATIVE-NEXT:    br label [[TMP16]]
+; IR-ITERATIVE:       16:
+; IR-ITERATIVE-NEXT:    [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
+; IR-ITERATIVE-NEXT:    [[TMP18:%.*]] = bitcast double [[TMP17]] to i64
+; IR-ITERATIVE-NEXT:    [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32
+; IR-ITERATIVE-NEXT:    [[TMP20:%.*]] = lshr i64 [[TMP18]], 32
+; IR-ITERATIVE-NEXT:    [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32
+; IR-ITERATIVE-NEXT:    [[TMP22:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP19]]) #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP23:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP21]]) #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP24:%.*]] = insertelement <2 x i32> poison, i32 [[TMP22]], i32 0
+; IR-ITERATIVE-NEXT:    [[TMP25:%.*]] = insertelement <2 x i32> [[TMP24]], i32 [[TMP23]], i32 1
+; IR-ITERATIVE-NEXT:    [[TMP26:%.*]] = bitcast <2 x i32> [[TMP25]] to double
+; IR-ITERATIVE-NEXT:    [[TMP27:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP28:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP27]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP29:%.*]] = select i1 [[TMP13]], double -0.000000e+00, double [[TMP28]]
+; IR-ITERATIVE-NEXT:    [[TMP30:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP26]], double [[TMP29]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    br label [[TMP31]]
+; IR-ITERATIVE:       31:
+; IR-ITERATIVE-NEXT:    [[TMP32:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP30]], [[TMP16]] ]
+; IR-ITERATIVE-NEXT:    ret double [[TMP32]]
+;
+; IR-DPP-LABEL: @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp(
+; IR-DPP-NEXT:    [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT:    br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP31:%.*]]
+; IR-DPP:       2:
+; IR-DPP-NEXT:    [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-DPP-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-DPP-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-DPP-NEXT:    [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
+; IR-DPP-NEXT:    [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-DPP-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
+; IR-DPP:       14:
+; IR-DPP-NEXT:    [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 8
+; IR-DPP-NEXT:    br label [[TMP16]]
+; IR-DPP:       16:
+; IR-DPP-NEXT:    [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
+; IR-DPP-NEXT:    [[TMP18:%.*]] = bitcast double [[TMP17]] to i64
+; IR-DPP-NEXT:    [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32
+; IR-DPP-NEXT:    [[TMP20:%.*]] = lshr i64 [[TMP18]], 32
+; IR-DPP-NEXT:    [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32
+; IR-DPP-NEXT:    [[TMP22:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP19]]) #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP23:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP21]]) #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP24:%.*]] = insertelement <2 x i32> poison, i32 [[TMP22]], i32 0
+; IR-DPP-NEXT:    [[TMP25:%.*]] = insertelement <2 x i32> [[TMP24]], i32 [[TMP23]], i32 1
+; IR-DPP-NEXT:    [[TMP26:%.*]] = bitcast <2 x i32> [[TMP25]] to double
+; IR-DPP-NEXT:    [[TMP27:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP28:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP27]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP29:%.*]] = select i1 [[TMP13]], double -0.000000e+00, double [[TMP28]]
+; IR-DPP-NEXT:    [[TMP30:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP26]], double [[TMP29]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT:    br label [[TMP31]]
+; IR-DPP:       31:
+; IR-DPP-NEXT:    [[TMP32:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP30]], [[TMP16]] ]
+; IR-DPP-NEXT:    ret double [[TMP32]]
 ;
   %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic
   ret double %result
@@ -1403,9 +1433,83 @@ define amdgpu_ps double @global_atomic__fmax_double_uni_address_div_value_agent_
 }
 
 define amdgpu_ps double @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #2 {
-; IR-LABEL: @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp(
-; IR-NEXT:    [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] monotonic, align 4
-; IR-NEXT:    ret double [[RESULT]]
+; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp(
+; IR-ITERATIVE-NEXT:    [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP31:%.*]]
+; IR-ITERATIVE:       2:
+; IR-ITERATIVE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-ITERATIVE-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-ITERATIVE-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-ITERATIVE-NEXT:    [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
+; IR-ITERATIVE-NEXT:    [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
+; IR-ITERATIVE:       14:
+; IR-ITERATIVE-NEXT:    [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] monotonic, align 4
+; IR-ITERATIVE-NEXT:    br label [[TMP16]]
+; IR-ITERATIVE:       16:
+; IR-ITERATIVE-NEXT:    [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
+; IR-ITERATIVE-NEXT:    [[TMP18:%.*]] = bitcast double [[TMP17]] to i64
+; IR-ITERATIVE-NEXT:    [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32
+; IR-ITERATIVE-NEXT:    [[TMP20:%.*]] = lshr i64 [[TMP18]], 32
+; IR-ITERATIVE-NEXT:    [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32
+; IR-ITERATIVE-NEXT:    [[TMP22:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP19]]) #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP23:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP21]]) #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP24:%.*]] = insertelement <2 x i32> poison, i32 [[TMP22]], i32 0
+; IR-ITERATIVE-NEXT:    [[TMP25:%.*]] = insertelement <2 x i32> [[TMP24]], i32 [[TMP23]], i32 1
+; IR-ITERATIVE-NEXT:    [[TMP26:%.*]] = bitcast <2 x i32> [[TMP25]] to double
+; IR-ITERATIVE-NEXT:    [[TMP27:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP28:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP27]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    [[TMP29:%.*]] = select i1 [[TMP13]], double -0.000000e+00, double [[TMP28]]
+; IR-ITERATIVE-NEXT:    [[TMP30:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP26]], double [[TMP29]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT:    br label [[TMP31]]
+; IR-ITERATIVE:       31:
+; IR-ITERATIVE-NEXT:    [[TMP32:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP30]], [[TMP16]] ]
+; IR-ITERATIVE-NEXT:    ret double [[TMP32]]
+;
+; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp(
+; IR-DPP-NEXT:    [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT:    br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP31:%.*]]
+; IR-DPP:       2:
+; IR-DPP-NEXT:    [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-DPP-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-DPP-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-DPP-NEXT:    [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
+; IR-DPP-NEXT:    [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-DPP-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
+; IR-DPP:       14:
+; IR-DPP-NEXT:    [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] monotonic, align 4
+; IR-DPP-NEXT:    br label [[TMP16]]
+; IR-DPP:       16:
+; IR-DPP-NEXT:    [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
+; IR-DPP-NEXT:    [[TMP18:%.*]] = bitcast double [[TMP17]] to i64
+; IR-DPP-NEXT:    [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32
+; IR-DPP-NEXT:    [[TMP20:%.*]] = lshr i64 [[TMP18]], 32
+; IR-DPP-NEXT:    [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32
+; IR-DPP-NEXT:    [[TMP22:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP19]]) #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP23:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP21]]) #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP24:%.*]] = insertelement <2 x i32> poison, i32 [[TMP22]], i32 0
+; IR-DPP-NEXT:    [[TMP25:%.*]] = insertelement <2 x i32> [[TMP24]], i32 [[TMP23]], i32 1
+; IR-DPP-NEXT:    [[TMP26:%.*]] = bitcast <2 x i32> [[TMP25]] to double
+; IR-DPP-NEXT:    [[TMP27:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP28:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP27]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT:    [[TMP29:%.*]] = select i1 [[TMP13]], double -0.000000e+00, double [[TMP28]]
+; IR-DPP-NEXT:    [[TMP30:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP26]], double [[TMP29]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT:    br label [[TMP31]]
+; IR-DPP:       31:
+; IR-DPP-NEXT:    [[TMP32:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP30]], [[TMP16]] ]
+; IR-DPP-NEXT:    ret double [[TMP32]]
 ;
   %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val monotonic, align 4
   ret double %result
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
index 9744bd42786ea..7c4684ee0c557 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
@@ -6979,91 +6979,83 @@ define void @local_atomic_fadd_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
 define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3) %ptrf, i32 %idx) {
 ; GFX12-LABEL: local_ds_fadd:
 ; GFX12:       ; %bb.0:
-; GFX12-NEXT:    v_mov_b32_e32 v1, 0x42280000
-; GFX12-NEXT:    s_mov_b32 s2, exec_lo
-; GFX12-NEXT:    s_brev_b32 s4, 1
-; GFX12-NEXT:    ; implicit-def: $vgpr0
-; GFX12-NEXT:  .LBB28_1: ; %ComputeLoop
-; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT:    s_ctz_i32_b32 s3, s2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT:    v_readlane_b32 s5, v1, s3
-; GFX12-NEXT:    s_lshl_b32 s6, 1, s3
-; GFX12-NEXT:    v_writelane_b32 v0, s4, s3
-; GFX12-NEXT:    s_and_not1_b32 s2, s2, s6
-; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT:    s_cmp_lg_u32 s2, 0
-; GFX12-NEXT:    s_add_f32 s4, s4, s5
-; GFX12-NEXT:    s_cbranch_scc1 .LBB28_1
-; GFX12-NEXT:  ; %bb.2: ; %ComputeEnd
-; GFX12-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
-; GFX12-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX12-NEXT:    s_load_b64 s[4:5], s[0:1], 0x8
+; GFX12-NEXT:    s_mov_b32 s6, exec_lo
 ; GFX12-NEXT:    ; implicit-def: $vgpr1
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_add_co_i32 s3, s3, 4
-; GFX12-NEXT:    s_and_saveexec_b32 s5, vcc_lo
-; GFX12-NEXT:    s_xor_b32 s5, exec_lo, s5
-; GFX12-NEXT:    s_cbranch_execz .LBB28_4
-; GFX12-NEXT:  ; %bb.3:
-; GFX12-NEXT:    s_lshl_b32 s6, s3, 3
+; GFX12-NEXT:    s_add_co_i32 s3, s5, 4
+; GFX12-NEXT:    s_and_saveexec_b32 s2, vcc_lo
+; GFX12-NEXT:    s_cbranch_execz .LBB28_2
+; GFX12-NEXT:  ; %bb.1:
+; GFX12-NEXT:    s_bcnt1_i32_b32 s5, s6
 ; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v1, s6
-; GFX12-NEXT:    ds_add_rtn_f32 v1, v1, v2
+; GFX12-NEXT:    v_cvt_f32_ubyte0_e32 v1, s5
+; GFX12-NEXT:    s_lshl_b32 s5, s3, 3
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT:    v_dual_mov_b32 v2, s5 :: v_dual_mul_f32 v1, 0x42280000, v1
+; GFX12-NEXT:    ds_add_rtn_f32 v1, v2, v1
 ; GFX12-NEXT:    s_wait_dscnt 0x0
 ; GFX12-NEXT:    global_inv scope:SCOPE_SE
-; GFX12-NEXT:  .LBB28_4:
-; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s5
+; GFX12-NEXT:  .LBB28_2:
+; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s2
 ; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT:    s_mov_b32 s6, exec_lo
+; GFX12-NEXT:    s_mov_b32 s7, exec_lo
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v1
-; GFX12-NEXT:    v_mbcnt_lo_u32_b32 v2, s6, 0
-; GFX12-NEXT:    s_mov_b32 s4, exec_lo
+; GFX12-NEXT:    v_mbcnt_lo_u32_b32 v2, s7, 0
+; GFX12-NEXT:    s_mov_b32 s6, exec_lo
 ; GFX12-NEXT:    v_cmpx_eq_u32_e32 0, v2
-; GFX12-NEXT:    s_cbranch_execz .LBB28_6
-; GFX12-NEXT:  ; %bb.5:
-; GFX12-NEXT:    s_bcnt1_i32_b32 s6, s6
-; GFX12-NEXT:    s_lshl_b32 s3, s3, 4
-; GFX12-NEXT:    v_cvt_f32_ubyte0_e32 v1, s6
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    v_dual_mov_b32 v2, s3 :: v_dual_mul_f32 v1, 0x42280000, v1
+; GFX12-NEXT:    s_cbranch_execz .LBB28_4
+; GFX12-NEXT:  ; %bb.3:
+; GFX12-NEXT:    s_bcnt1_i32_b32 s2, s7
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    v_cvt_f32_ubyte0_e32 v1, s2
+; GFX12-NEXT:    s_lshl_b32 s2, s3, 4
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mul_f32 v1, 0x42280000, v1
 ; GFX12-NEXT:    ds_add_f32 v2, v1
 ; GFX12-NEXT:    s_wait_dscnt 0x0
 ; GFX12-NEXT:    global_inv scope:SCOPE_SE
-; GFX12-NEXT:  .LBB28_6:
-; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:  .LBB28_4:
+; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s6
+; GFX12-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
+; GFX12-NEXT:    s_mov_b32 s3, exec_lo
+; GFX12-NEXT:    s_brev_b32 s2, 1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_mul_f32_e32 v0, 0x42280000, v0
+; GFX12-NEXT:    v_cndmask_b32_e64 v0, v0, 0x80000000, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_add_f32_e32 v1, s5, v0
-; GFX12-NEXT:    s_mov_b32 s4, exec_lo
-; GFX12-NEXT:    s_brev_b32 s3, 1
 ; GFX12-NEXT:    ; implicit-def: $vgpr0
-; GFX12-NEXT:  .LBB28_7: ; %ComputeLoop1
+; GFX12-NEXT:  .LBB28_5: ; %ComputeLoop
 ; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT:    s_ctz_i32_b32 s5, s4
+; GFX12-NEXT:    s_ctz_i32_b32 s5, s3
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
 ; GFX12-NEXT:    v_readlane_b32 s6, v1, s5
 ; GFX12-NEXT:    s_lshl_b32 s7, 1, s5
-; GFX12-NEXT:    v_writelane_b32 v0, s3, s5
-; GFX12-NEXT:    s_and_not1_b32 s4, s4, s7
+; GFX12-NEXT:    v_writelane_b32 v0, s2, s5
+; GFX12-NEXT:    s_and_not1_b32 s3, s3, s7
 ; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT:    s_cmp_lg_u32 s4, 0
-; GFX12-NEXT:    s_add_f32 s3, s3, s6
-; GFX12-NEXT:    s_cbranch_scc1 .LBB28_7
-; GFX12-NEXT:  ; %bb.8: ; %ComputeEnd2
+; GFX12-NEXT:    s_cmp_lg_u32 s3, 0
+; GFX12-NEXT:    s_add_f32 s2, s2, s6
+; GFX12-NEXT:    s_cbranch_scc1 .LBB28_5
+; GFX12-NEXT:  ; %bb.6: ; %ComputeEnd
 ; GFX12-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
 ; GFX12-NEXT:    ; implicit-def: $vgpr1
-; GFX12-NEXT:    s_and_saveexec_b32 s4, vcc_lo
-; GFX12-NEXT:    s_xor_b32 s4, exec_lo, s4
-; GFX12-NEXT:    s_cbranch_execz .LBB28_10
-; GFX12-NEXT:  ; %bb.9:
-; GFX12-NEXT:    v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s3
+; GFX12-NEXT:    s_and_saveexec_b32 s3, vcc_lo
+; GFX12-NEXT:    s_xor_b32 s3, exec_lo, s3
+; GFX12-NEXT:    s_cbranch_execz .LBB28_8
+; GFX12-NEXT:  ; %bb.7:
+; GFX12-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_mov_b32 v2, s2
 ; GFX12-NEXT:    ds_add_rtn_f32 v1, v1, v2
 ; GFX12-NEXT:    s_wait_dscnt 0x0
 ; GFX12-NEXT:    global_inv scope:SCOPE_SE
-; GFX12-NEXT:  .LBB28_10:
-; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:  .LBB28_8:
+; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s3
 ; GFX12-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX12-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
@@ -7076,87 +7068,76 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
 ;
 ; GFX940-LABEL: local_ds_fadd:
 ; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
 ; GFX940-NEXT:    s_mov_b64 s[2:3], exec
-; GFX940-NEXT:    v_bfrev_b32_e32 v1, 1
-; GFX940-NEXT:    v_mov_b32_e32 v2, 0x42280000
-; GFX940-NEXT:    ; implicit-def: $vgpr0
-; GFX940-NEXT:  .LBB28_1: ; %ComputeLoop
-; GFX940-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX940-NEXT:    s_ff1_i32_b64 s6, s[2:3]
-; GFX940-NEXT:    s_lshl_b64 s[4:5], 1, s6
-; GFX940-NEXT:    v_readfirstlane_b32 s7, v1
-; GFX940-NEXT:    v_readlane_b32 s8, v2, s6
-; GFX940-NEXT:    s_mov_b32 m0, s6
-; GFX940-NEXT:    s_andn2_b64 s[2:3], s[2:3], s[4:5]
-; GFX940-NEXT:    v_writelane_b32 v0, s7, m0
-; GFX940-NEXT:    s_cmp_lg_u64 s[2:3], 0
-; GFX940-NEXT:    v_add_f32_e32 v1, s8, v1
-; GFX940-NEXT:    s_cbranch_scc1 .LBB28_1
-; GFX940-NEXT:  ; %bb.2: ; %ComputeEnd
-; GFX940-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
-; GFX940-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX940-NEXT:    v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX940-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
-; GFX940-NEXT:    ; implicit-def: $vgpr2
+; GFX940-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX940-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX940-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    s_add_i32 s5, s5, 4
+; GFX940-NEXT:    ; implicit-def: $vgpr1
+; GFX940-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; GFX940-NEXT:    s_cbranch_execz .LBB28_2
+; GFX940-NEXT:  ; %bb.1:
+; GFX940-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
+; GFX940-NEXT:    s_lshl_b32 s8, s5, 3
+; GFX940-NEXT:    v_cvt_f32_ubyte0_e32 v1, s2
+; GFX940-NEXT:    v_mul_f32_e32 v1, 0x42280000, v1
+; GFX940-NEXT:    v_mov_b32_e32 v2, s8
+; GFX940-NEXT:    ds_add_rtn_f32 v1, v2, v1
 ; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NEXT:    s_add_i32 s3, s3, 4
-; GFX940-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX940-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX940-NEXT:  .LBB28_2:
+; GFX940-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX940-NEXT:    s_mov_b64 s[8:9], exec
+; GFX940-NEXT:    v_readfirstlane_b32 s10, v1
+; GFX940-NEXT:    v_mbcnt_lo_u32_b32 v1, s8, 0
+; GFX940-NEXT:    v_mbcnt_hi_u32_b32 v1, s9, v1
+; GFX940-NEXT:    v_cmp_eq_u32_e64 s[2:3], 0, v1
+; GFX940-NEXT:    s_and_saveexec_b64 s[6:7], s[2:3]
 ; GFX940-NEXT:    s_cbranch_execz .LBB28_4
 ; GFX940-NEXT:  ; %bb.3:
-; GFX940-NEXT:    s_lshl_b32 s6, s3, 3
-; GFX940-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-NEXT:    ds_add_rtn_f32 v2, v2, v1
-; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NEXT:  .LBB28_4:
-; GFX940-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX940-NEXT:    s_mov_b64 s[6:7], exec
-; GFX940-NEXT:    v_mbcnt_lo_u32_b32 v1, s6, 0
-; GFX940-NEXT:    v_mbcnt_hi_u32_b32 v1, s7, v1
-; GFX940-NEXT:    v_readfirstlane_b32 s8, v2
-; GFX940-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX940-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX940-NEXT:    s_cbranch_execz .LBB28_6
-; GFX940-NEXT:  ; %bb.5:
-; GFX940-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
-; GFX940-NEXT:    v_cvt_f32_ubyte0_e32 v1, s6
-; GFX940-NEXT:    s_lshl_b32 s3, s3, 4
+; GFX940-NEXT:    s_bcnt1_i32_b64 s2, s[8:9]
+; GFX940-NEXT:    v_cvt_f32_ubyte0_e32 v1, s2
+; GFX940-NEXT:    s_lshl_b32 s2, s5, 4
 ; GFX940-NEXT:    v_mul_f32_e32 v1, 0x42280000, v1
-; GFX940-NEXT:    v_mov_b32_e32 v2, s3
+; GFX940-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX940-NEXT:    ds_add_f32 v2, v1
 ; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NEXT:  .LBB28_6:
-; GFX940-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX940-NEXT:    s_mov_b64 s[4:5], exec
-; GFX940-NEXT:    v_add_f32_e32 v2, s8, v0
+; GFX940-NEXT:  .LBB28_4:
+; GFX940-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX940-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
+; GFX940-NEXT:    v_mul_f32_e32 v0, 0x42280000, v0
 ; GFX940-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX940-NEXT:    s_mov_b64 s[2:3], exec
+; GFX940-NEXT:    v_add_f32_e32 v2, s10, v0
 ; GFX940-NEXT:    ; implicit-def: $vgpr0
-; GFX940-NEXT:  .LBB28_7: ; %ComputeLoop1
+; GFX940-NEXT:  .LBB28_5: ; %ComputeLoop
 ; GFX940-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX940-NEXT:    s_ff1_i32_b64 s3, s[4:5]
-; GFX940-NEXT:    s_lshl_b64 s[6:7], 1, s3
+; GFX940-NEXT:    s_ff1_i32_b64 s5, s[2:3]
+; GFX940-NEXT:    s_lshl_b64 s[6:7], 1, s5
 ; GFX940-NEXT:    v_readfirstlane_b32 s8, v1
-; GFX940-NEXT:    v_readlane_b32 s9, v2, s3
-; GFX940-NEXT:    s_mov_b32 m0, s3
-; GFX940-NEXT:    s_andn2_b64 s[4:5], s[4:5], s[6:7]
+; GFX940-NEXT:    v_readlane_b32 s9, v2, s5
+; GFX940-NEXT:    s_mov_b32 m0, s5
+; GFX940-NEXT:    s_andn2_b64 s[2:3], s[2:3], s[6:7]
 ; GFX940-NEXT:    v_writelane_b32 v0, s8, m0
-; GFX940-NEXT:    s_cmp_lg_u64 s[4:5], 0
+; GFX940-NEXT:    s_cmp_lg_u64 s[2:3], 0
 ; GFX940-NEXT:    v_add_f32_e32 v1, s9, v1
-; GFX940-NEXT:    s_cbranch_scc1 .LBB28_7
-; GFX940-NEXT:  ; %bb.8: ; %ComputeEnd2
+; GFX940-NEXT:    s_cbranch_scc1 .LBB28_5
+; GFX940-NEXT:  ; %bb.6: ; %ComputeEnd
 ; GFX940-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
 ; GFX940-NEXT:    v_mbcnt_hi_u32_b32 v2, exec_hi, v2
 ; GFX940-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
 ; GFX940-NEXT:    ; implicit-def: $vgpr2
-; GFX940-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX940-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GFX940-NEXT:    s_cbranch_execz .LBB28_10
-; GFX940-NEXT:  ; %bb.9:
-; GFX940-NEXT:    v_mov_b32_e32 v2, s2
+; GFX940-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX940-NEXT:    s_xor_b64 s[2:3], exec, s[2:3]
+; GFX940-NEXT:    s_cbranch_execz .LBB28_8
+; GFX940-NEXT:  ; %bb.7:
+; GFX940-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX940-NEXT:    ds_add_rtn_f32 v2, v2, v1
 ; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NEXT:  .LBB28_10:
-; GFX940-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX940-NEXT:  .LBB28_8:
+; GFX940-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX940-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
 ; GFX940-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX940-NEXT:    v_mov_b32_e32 v1, 0
@@ -7168,95 +7149,85 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
 ;
 ; GFX11-LABEL: local_ds_fadd:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    v_bfrev_b32_e32 v1, 1
-; GFX11-NEXT:    v_mov_b32_e32 v2, 0x42280000
-; GFX11-NEXT:    s_mov_b32 s2, exec_lo
-; GFX11-NEXT:    ; implicit-def: $vgpr0
-; GFX11-NEXT:  .LBB28_1: ; %ComputeLoop
-; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    s_load_b64 s[4:5], s[0:1], 0x8
+; GFX11-NEXT:    s_mov_b32 s6, exec_lo
+; GFX11-NEXT:    ; implicit-def: $vgpr1
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    s_ctz_i32_b32 s3, s2
-; GFX11-NEXT:    v_readfirstlane_b32 s4, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT:    v_readlane_b32 s5, v2, s3
-; GFX11-NEXT:    s_lshl_b32 s6, 1, s3
-; GFX11-NEXT:    s_and_not1_b32 s2, s2, s6
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_writelane_b32 v0, s4, s3
-; GFX11-NEXT:    v_add_f32_e32 v1, s5, v1
-; GFX11-NEXT:    s_cmp_lg_u32 s2, 0
-; GFX11-NEXT:    s_cbranch_scc1 .LBB28_1
-; GFX11-NEXT:  ; %bb.2: ; %ComputeEnd
-; GFX11-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
-; GFX11-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v2
-; GFX11-NEXT:    ; implicit-def: $vgpr2
+; GFX11-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_add_i32 s3, s3, 4
-; GFX11-NEXT:    s_and_saveexec_b32 s4, vcc_lo
-; GFX11-NEXT:    s_xor_b32 s4, exec_lo, s4
-; GFX11-NEXT:    s_cbranch_execz .LBB28_4
-; GFX11-NEXT:  ; %bb.3:
-; GFX11-NEXT:    s_lshl_b32 s5, s3, 3
+; GFX11-NEXT:    s_add_i32 s3, s5, 4
+; GFX11-NEXT:    s_and_saveexec_b32 s2, vcc_lo
+; GFX11-NEXT:    s_cbranch_execz .LBB28_2
+; GFX11-NEXT:  ; %bb.1:
+; GFX11-NEXT:    s_bcnt1_i32_b32 s5, s6
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT:    v_mov_b32_e32 v2, s5
-; GFX11-NEXT:    ds_add_rtn_f32 v2, v2, v1
+; GFX11-NEXT:    v_cvt_f32_ubyte0_e32 v1, s5
+; GFX11-NEXT:    s_lshl_b32 s5, s3, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v2, s5 :: v_dual_mul_f32 v1, 0x42280000, v1
+; GFX11-NEXT:    ds_add_rtn_f32 v1, v2, v1
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    buffer_gl0_inv
-; GFX11-NEXT:  .LBB28_4:
-; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:  .LBB28_2:
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s2
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    s_mov_b32 s7, exec_lo
+; GFX11-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX11-NEXT:    v_mbcnt_lo_u32_b32 v2, s7, 0
 ; GFX11-NEXT:    s_mov_b32 s6, exec_lo
-; GFX11-NEXT:    v_readfirstlane_b32 s4, v2
-; GFX11-NEXT:    v_mbcnt_lo_u32_b32 v1, s6, 0
-; GFX11-NEXT:    s_mov_b32 s5, exec_lo
-; GFX11-NEXT:    v_cmpx_eq_u32_e32 0, v1
-; GFX11-NEXT:    s_cbranch_execz .LBB28_6
-; GFX11-NEXT:  ; %bb.5:
-; GFX11-NEXT:    s_bcnt1_i32_b32 s6, s6
-; GFX11-NEXT:    s_lshl_b32 s3, s3, 4
-; GFX11-NEXT:    v_cvt_f32_ubyte0_e32 v1, s6
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_dual_mov_b32 v2, s3 :: v_dual_mul_f32 v1, 0x42280000, v1
+; GFX11-NEXT:    v_cmpx_eq_u32_e32 0, v2
+; GFX11-NEXT:    s_cbranch_execz .LBB28_4
+; GFX11-NEXT:  ; %bb.3:
+; GFX11-NEXT:    s_bcnt1_i32_b32 s2, s7
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_cvt_f32_ubyte0_e32 v1, s2
+; GFX11-NEXT:    s_lshl_b32 s2, s3, 4
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mul_f32 v1, 0x42280000, v1
 ; GFX11-NEXT:    ds_add_f32 v2, v1
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    buffer_gl0_inv
-; GFX11-NEXT:  .LBB28_6:
-; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT:    v_add_f32_e32 v2, s4, v0
+; GFX11-NEXT:  .LBB28_4:
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s6
+; GFX11-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
 ; GFX11-NEXT:    v_bfrev_b32_e32 v1, 1
-; GFX11-NEXT:    s_mov_b32 s3, exec_lo
+; GFX11-NEXT:    s_mov_b32 s2, exec_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_mul_f32_e32 v0, 0x42280000, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, 0x80000000, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_add_f32_e32 v2, s5, v0
 ; GFX11-NEXT:    ; implicit-def: $vgpr0
-; GFX11-NEXT:  .LBB28_7: ; %ComputeLoop1
+; GFX11-NEXT:  .LBB28_5: ; %ComputeLoop
 ; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    s_ctz_i32_b32 s4, s3
+; GFX11-NEXT:    s_ctz_i32_b32 s3, s2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_readfirstlane_b32 s5, v1
-; GFX11-NEXT:    v_readlane_b32 s6, v2, s4
-; GFX11-NEXT:    s_lshl_b32 s7, 1, s4
+; GFX11-NEXT:    v_readlane_b32 s6, v2, s3
+; GFX11-NEXT:    s_lshl_b32 s7, 1, s3
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    s_and_not1_b32 s3, s3, s7
-; GFX11-NEXT:    v_writelane_b32 v0, s5, s4
+; GFX11-NEXT:    s_and_not1_b32 s2, s2, s7
+; GFX11-NEXT:    v_writelane_b32 v0, s5, s3
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-NEXT:    v_add_f32_e32 v1, s6, v1
-; GFX11-NEXT:    s_cmp_lg_u32 s3, 0
-; GFX11-NEXT:    s_cbranch_scc1 .LBB28_7
-; GFX11-NEXT:  ; %bb.8: ; %ComputeEnd2
+; GFX11-NEXT:    s_cmp_lg_u32 s2, 0
+; GFX11-NEXT:    s_cbranch_scc1 .LBB28_5
+; GFX11-NEXT:  ; %bb.6: ; %ComputeEnd
 ; GFX11-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v2
 ; GFX11-NEXT:    ; implicit-def: $vgpr2
-; GFX11-NEXT:    s_and_saveexec_b32 s3, vcc_lo
-; GFX11-NEXT:    s_xor_b32 s3, exec_lo, s3
-; GFX11-NEXT:    s_cbranch_execz .LBB28_10
-; GFX11-NEXT:  ; %bb.9:
-; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    s_and_saveexec_b32 s2, vcc_lo
+; GFX11-NEXT:    s_xor_b32 s2, exec_lo, s2
+; GFX11-NEXT:    s_cbranch_execz .LBB28_8
+; GFX11-NEXT:  ; %bb.7:
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX11-NEXT:    ds_add_rtn_f32 v2, v2, v1
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    buffer_gl0_inv
-; GFX11-NEXT:  .LBB28_10:
-; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s3
+; GFX11-NEXT:  .LBB28_8:
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s2
 ; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX11-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
@@ -7269,90 +7240,80 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
 ;
 ; GFX10-LABEL: local_ds_fadd:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_bfrev_b32_e32 v1, 1
-; GFX10-NEXT:    v_mov_b32_e32 v2, 0x42280000
-; GFX10-NEXT:    s_mov_b32 s2, exec_lo
-; GFX10-NEXT:    ; implicit-def: $vgpr0
-; GFX10-NEXT:  .LBB28_1: ; %ComputeLoop
-; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT:    s_ff1_i32_b32 s3, s2
-; GFX10-NEXT:    v_readfirstlane_b32 s4, v1
-; GFX10-NEXT:    v_readlane_b32 s5, v2, s3
-; GFX10-NEXT:    s_lshl_b32 s6, 1, s3
-; GFX10-NEXT:    s_andn2_b32 s2, s2, s6
-; GFX10-NEXT:    v_writelane_b32 v0, s4, s3
-; GFX10-NEXT:    v_add_f32_e32 v1, s5, v1
-; GFX10-NEXT:    s_cmp_lg_u32 s2, 0
-; GFX10-NEXT:    s_cbranch_scc1 .LBB28_1
-; GFX10-NEXT:  ; %bb.2: ; %ComputeEnd
-; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
-; GFX10-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    ; implicit-def: $vgpr2
+; GFX10-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
+; GFX10-NEXT:    s_mov_b32 s6, exec_lo
+; GFX10-NEXT:    ; implicit-def: $vgpr1
+; GFX10-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    s_add_i32 s3, s3, 4
-; GFX10-NEXT:    s_and_saveexec_b32 s4, vcc_lo
-; GFX10-NEXT:    s_xor_b32 s4, exec_lo, s4
-; GFX10-NEXT:    s_cbranch_execz .LBB28_4
-; GFX10-NEXT:  ; %bb.3:
+; GFX10-NEXT:    s_add_i32 s3, s5, 4
+; GFX10-NEXT:    s_and_saveexec_b32 s2, vcc_lo
+; GFX10-NEXT:    s_cbranch_execz .LBB28_2
+; GFX10-NEXT:  ; %bb.1:
+; GFX10-NEXT:    s_bcnt1_i32_b32 s5, s6
+; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v1, s5
 ; GFX10-NEXT:    s_lshl_b32 s5, s3, 3
 ; GFX10-NEXT:    v_mov_b32_e32 v2, s5
-; GFX10-NEXT:    ds_add_rtn_f32 v2, v2, v1
+; GFX10-NEXT:    v_mul_f32_e32 v1, 0x42280000, v1
+; GFX10-NEXT:    ds_add_rtn_f32 v1, v2, v1
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    buffer_gl0_inv
-; GFX10-NEXT:  .LBB28_4:
+; GFX10-NEXT:  .LBB28_2:
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT:    s_mov_b32 s6, exec_lo
-; GFX10-NEXT:    v_readfirstlane_b32 s4, v2
-; GFX10-NEXT:    v_mbcnt_lo_u32_b32 v1, s6, 0
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX10-NEXT:    s_and_saveexec_b32 s5, vcc_lo
-; GFX10-NEXT:    s_cbranch_execz .LBB28_6
-; GFX10-NEXT:  ; %bb.5:
-; GFX10-NEXT:    s_bcnt1_i32_b32 s6, s6
-; GFX10-NEXT:    s_lshl_b32 s3, s3, 4
-; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v1, s6
-; GFX10-NEXT:    v_mov_b32_e32 v2, s3
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s2
+; GFX10-NEXT:    s_mov_b32 s7, exec_lo
+; GFX10-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX10-NEXT:    v_mbcnt_lo_u32_b32 v2, s7, 0
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s2, 0, v2
+; GFX10-NEXT:    s_and_saveexec_b32 s6, s2
+; GFX10-NEXT:    s_cbranch_execz .LBB28_4
+; GFX10-NEXT:  ; %bb.3:
+; GFX10-NEXT:    s_bcnt1_i32_b32 s2, s7
+; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v1, s2
+; GFX10-NEXT:    s_lshl_b32 s2, s3, 4
+; GFX10-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX10-NEXT:    v_mul_f32_e32 v1, 0x42280000, v1
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    ds_add_f32 v2, v1
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    buffer_gl0_inv
-; GFX10-NEXT:  .LBB28_6:
+; GFX10-NEXT:  .LBB28_4:
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT:    v_add_f32_e32 v2, s4, v0
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s6
+; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
 ; GFX10-NEXT:    v_bfrev_b32_e32 v1, 1
-; GFX10-NEXT:    s_mov_b32 s3, exec_lo
+; GFX10-NEXT:    s_mov_b32 s2, exec_lo
+; GFX10-NEXT:    v_mul_f32_e32 v0, 0x42280000, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, 0x80000000, vcc_lo
+; GFX10-NEXT:    v_add_f32_e32 v2, s5, v0
 ; GFX10-NEXT:    ; implicit-def: $vgpr0
-; GFX10-NEXT:  .LBB28_7: ; %ComputeLoop1
+; GFX10-NEXT:  .LBB28_5: ; %ComputeLoop
 ; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT:    s_ff1_i32_b32 s4, s3
+; GFX10-NEXT:    s_ff1_i32_b32 s3, s2
 ; GFX10-NEXT:    v_readfirstlane_b32 s5, v1
-; GFX10-NEXT:    v_readlane_b32 s6, v2, s4
-; GFX10-NEXT:    s_lshl_b32 s7, 1, s4
-; GFX10-NEXT:    s_andn2_b32 s3, s3, s7
-; GFX10-NEXT:    v_writelane_b32 v0, s5, s4
+; GFX10-NEXT:    v_readlane_b32 s6, v2, s3
+; GFX10-NEXT:    s_lshl_b32 s7, 1, s3
+; GFX10-NEXT:    s_andn2_b32 s2, s2, s7
+; GFX10-NEXT:    v_writelane_b32 v0, s5, s3
 ; GFX10-NEXT:    v_add_f32_e32 v1, s6, v1
-; GFX10-NEXT:    s_cmp_lg_u32 s3, 0
-; GFX10-NEXT:    s_cbranch_scc1 .LBB28_7
-; GFX10-NEXT:  ; %bb.8: ; %ComputeEnd2
+; GFX10-NEXT:    s_cmp_lg_u32 s2, 0
+; GFX10-NEXT:    s_cbranch_scc1 .LBB28_5
+; GFX10-NEXT:  ; %bb.6: ; %ComputeEnd
 ; GFX10-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v2
 ; GFX10-NEXT:    ; implicit-def: $vgpr2
-; GFX10-NEXT:    s_and_saveexec_b32 s3, vcc_lo
-; GFX10-NEXT:    s_xor_b32 s3, exec_lo, s3
-; GFX10-NEXT:    s_cbranch_execz .LBB28_10
-; GFX10-NEXT:  ; %bb.9:
-; GFX10-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-NEXT:    s_and_saveexec_b32 s2, vcc_lo
+; GFX10-NEXT:    s_xor_b32 s2, exec_lo, s2
+; GFX10-NEXT:    s_cbranch_execz .LBB28_8
+; GFX10-NEXT:  ; %bb.7:
+; GFX10-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    ds_add_rtn_f32 v2, v2, v1
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    buffer_gl0_inv
-; GFX10-NEXT:  .LBB28_10:
+; GFX10-NEXT:  .LBB28_8:
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s3
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s2
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
 ; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
@@ -7363,87 +7324,76 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
 ;
 ; GFX90A-LABEL: local_ds_fadd:
 ; GFX90A:       ; %bb.0:
+; GFX90A-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
 ; GFX90A-NEXT:    s_mov_b64 s[2:3], exec
-; GFX90A-NEXT:    v_bfrev_b32_e32 v1, 1
-; GFX90A-NEXT:    v_mov_b32_e32 v2, 0x42280000
-; GFX90A-NEXT:    ; implicit-def: $vgpr0
-; GFX90A-NEXT:  .LBB28_1: ; %ComputeLoop
-; GFX90A-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT:    s_ff1_i32_b64 s6, s[2:3]
-; GFX90A-NEXT:    s_lshl_b64 s[4:5], 1, s6
-; GFX90A-NEXT:    v_readfirstlane_b32 s7, v1
-; GFX90A-NEXT:    v_readlane_b32 s8, v2, s6
-; GFX90A-NEXT:    s_mov_b32 m0, s6
-; GFX90A-NEXT:    s_andn2_b64 s[2:3], s[2:3], s[4:5]
-; GFX90A-NEXT:    v_writelane_b32 v0, s7, m0
-; GFX90A-NEXT:    s_cmp_lg_u64 s[2:3], 0
-; GFX90A-NEXT:    v_add_f32_e32 v1, s8, v1
-; GFX90A-NEXT:    s_cbranch_scc1 .LBB28_1
-; GFX90A-NEXT:  ; %bb.2: ; %ComputeEnd
-; GFX90A-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
-; GFX90A-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
-; GFX90A-NEXT:    ; implicit-def: $vgpr2
+; GFX90A-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT:    s_add_i32 s3, s3, 4
-; GFX90A-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX90A-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX90A-NEXT:    s_add_i32 s5, s5, 4
+; GFX90A-NEXT:    ; implicit-def: $vgpr1
+; GFX90A-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; GFX90A-NEXT:    s_cbranch_execz .LBB28_2
+; GFX90A-NEXT:  ; %bb.1:
+; GFX90A-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
+; GFX90A-NEXT:    s_lshl_b32 s8, s5, 3
+; GFX90A-NEXT:    v_cvt_f32_ubyte0_e32 v1, s2
+; GFX90A-NEXT:    v_mul_f32_e32 v1, 0x42280000, v1
+; GFX90A-NEXT:    v_mov_b32_e32 v2, s8
+; GFX90A-NEXT:    ds_add_rtn_f32 v1, v2, v1
+; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX90A-NEXT:  .LBB28_2:
+; GFX90A-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT:    s_mov_b64 s[8:9], exec
+; GFX90A-NEXT:    v_readfirstlane_b32 s10, v1
+; GFX90A-NEXT:    v_mbcnt_lo_u32_b32 v1, s8, 0
+; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v1, s9, v1
+; GFX90A-NEXT:    v_cmp_eq_u32_e64 s[2:3], 0, v1
+; GFX90A-NEXT:    s_and_saveexec_b64 s[6:7], s[2:3]
 ; GFX90A-NEXT:    s_cbranch_execz .LBB28_4
 ; GFX90A-NEXT:  ; %bb.3:
-; GFX90A-NEXT:    s_lshl_b32 s6, s3, 3
-; GFX90A-NEXT:    v_mov_b32_e32 v2, s6
-; GFX90A-NEXT:    ds_add_rtn_f32 v2, v2, v1
-; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT:  .LBB28_4:
-; GFX90A-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT:    s_mov_b64 s[6:7], exec
-; GFX90A-NEXT:    v_mbcnt_lo_u32_b32 v1, s6, 0
-; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v1, s7, v1
-; GFX90A-NEXT:    v_readfirstlane_b32 s8, v2
-; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX90A-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX90A-NEXT:    s_cbranch_execz .LBB28_6
-; GFX90A-NEXT:  ; %bb.5:
-; GFX90A-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
-; GFX90A-NEXT:    v_cvt_f32_ubyte0_e32 v1, s6
-; GFX90A-NEXT:    s_lshl_b32 s3, s3, 4
+; GFX90A-NEXT:    s_bcnt1_i32_b64 s2, s[8:9]
+; GFX90A-NEXT:    v_cvt_f32_ubyte0_e32 v1, s2
+; GFX90A-NEXT:    s_lshl_b32 s2, s5, 4
 ; GFX90A-NEXT:    v_mul_f32_e32 v1, 0x42280000, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v2, s3
+; GFX90A-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX90A-NEXT:    ds_add_f32 v2, v1
 ; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT:  .LBB28_6:
-; GFX90A-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT:    s_mov_b64 s[4:5], exec
-; GFX90A-NEXT:    v_add_f32_e32 v2, s8, v0
+; GFX90A-NEXT:  .LBB28_4:
+; GFX90A-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
+; GFX90A-NEXT:    v_mul_f32_e32 v0, 0x42280000, v0
 ; GFX90A-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX90A-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX90A-NEXT:    s_mov_b64 s[2:3], exec
+; GFX90A-NEXT:    v_add_f32_e32 v2, s10, v0
 ; GFX90A-NEXT:    ; implicit-def: $vgpr0
-; GFX90A-NEXT:  .LBB28_7: ; %ComputeLoop1
+; GFX90A-NEXT:  .LBB28_5: ; %ComputeLoop
 ; GFX90A-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT:    s_ff1_i32_b64 s3, s[4:5]
-; GFX90A-NEXT:    s_lshl_b64 s[6:7], 1, s3
+; GFX90A-NEXT:    s_ff1_i32_b64 s5, s[2:3]
+; GFX90A-NEXT:    s_lshl_b64 s[6:7], 1, s5
 ; GFX90A-NEXT:    v_readfirstlane_b32 s8, v1
-; GFX90A-NEXT:    v_readlane_b32 s9, v2, s3
-; GFX90A-NEXT:    s_mov_b32 m0, s3
-; GFX90A-NEXT:    s_andn2_b64 s[4:5], s[4:5], s[6:7]
+; GFX90A-NEXT:    v_readlane_b32 s9, v2, s5
+; GFX90A-NEXT:    s_mov_b32 m0, s5
+; GFX90A-NEXT:    s_andn2_b64 s[2:3], s[2:3], s[6:7]
 ; GFX90A-NEXT:    v_writelane_b32 v0, s8, m0
-; GFX90A-NEXT:    s_cmp_lg_u64 s[4:5], 0
+; GFX90A-NEXT:    s_cmp_lg_u64 s[2:3], 0
 ; GFX90A-NEXT:    v_add_f32_e32 v1, s9, v1
-; GFX90A-NEXT:    s_cbranch_scc1 .LBB28_7
-; GFX90A-NEXT:  ; %bb.8: ; %ComputeEnd2
+; GFX90A-NEXT:    s_cbranch_scc1 .LBB28_5
+; GFX90A-NEXT:  ; %bb.6: ; %ComputeEnd
 ; GFX90A-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
 ; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v2, exec_hi, v2
 ; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
 ; GFX90A-NEXT:    ; implicit-def: $vgpr2
-; GFX90A-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX90A-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GFX90A-NEXT:    s_cbranch_execz .LBB28_10
-; GFX90A-NEXT:  ; %bb.9:
-; GFX90A-NEXT:    v_mov_b32_e32 v2, s2
+; GFX90A-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX90A-NEXT:    s_xor_b64 s[2:3], exec, s[2:3]
+; GFX90A-NEXT:    s_cbranch_execz .LBB28_8
+; GFX90A-NEXT:  ; %bb.7:
+; GFX90A-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX90A-NEXT:    ds_add_rtn_f32 v2, v2, v1
 ; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT:  .LBB28_10:
-; GFX90A-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT:  .LBB28_8:
+; GFX90A-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
 ; GFX90A-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX90A-NEXT:    v_mov_b32_e32 v1, 0
@@ -7454,87 +7404,76 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
 ;
 ; GFX908-LABEL: local_ds_fadd:
 ; GFX908:       ; %bb.0:
+; GFX908-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
 ; GFX908-NEXT:    s_mov_b64 s[2:3], exec
-; GFX908-NEXT:    v_bfrev_b32_e32 v1, 1
-; GFX908-NEXT:    v_mov_b32_e32 v2, 0x42280000
-; GFX908-NEXT:    ; implicit-def: $vgpr0
-; GFX908-NEXT:  .LBB28_1: ; %ComputeLoop
-; GFX908-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT:    s_ff1_i32_b64 s6, s[2:3]
-; GFX908-NEXT:    s_lshl_b64 s[4:5], 1, s6
-; GFX908-NEXT:    v_readfirstlane_b32 s7, v1
-; GFX908-NEXT:    v_readlane_b32 s8, v2, s6
-; GFX908-NEXT:    s_mov_b32 m0, s6
-; GFX908-NEXT:    s_andn2_b64 s[2:3], s[2:3], s[4:5]
-; GFX908-NEXT:    v_writelane_b32 v0, s7, m0
-; GFX908-NEXT:    s_cmp_lg_u64 s[2:3], 0
-; GFX908-NEXT:    v_add_f32_e32 v1, s8, v1
-; GFX908-NEXT:    s_cbranch_scc1 .LBB28_1
-; GFX908-NEXT:  ; %bb.2: ; %ComputeEnd
-; GFX908-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
-; GFX908-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX908-NEXT:    v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX908-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
-; GFX908-NEXT:    ; implicit-def: $vgpr2
+; GFX908-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX908-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX908-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX908-NEXT:    s_add_i32 s5, s5, 4
+; GFX908-NEXT:    ; implicit-def: $vgpr1
+; GFX908-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; GFX908-NEXT:    s_cbranch_execz .LBB28_2
+; GFX908-NEXT:  ; %bb.1:
+; GFX908-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
+; GFX908-NEXT:    s_lshl_b32 s8, s5, 3
+; GFX908-NEXT:    v_cvt_f32_ubyte0_e32 v1, s2
+; GFX908-NEXT:    v_mul_f32_e32 v1, 0x42280000, v1
+; GFX908-NEXT:    v_mov_b32_e32 v2, s8
+; GFX908-NEXT:    ds_add_rtn_f32 v1, v2, v1
 ; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX908-NEXT:    s_add_i32 s3, s3, 4
-; GFX908-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX908-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX908-NEXT:  .LBB28_2:
+; GFX908-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX908-NEXT:    s_mov_b64 s[8:9], exec
+; GFX908-NEXT:    v_readfirstlane_b32 s10, v1
+; GFX908-NEXT:    v_mbcnt_lo_u32_b32 v1, s8, 0
+; GFX908-NEXT:    v_mbcnt_hi_u32_b32 v1, s9, v1
+; GFX908-NEXT:    v_cmp_eq_u32_e64 s[2:3], 0, v1
+; GFX908-NEXT:    s_and_saveexec_b64 s[6:7], s[2:3]
 ; GFX908-NEXT:    s_cbranch_execz .LBB28_4
 ; GFX908-NEXT:  ; %bb.3:
-; GFX908-NEXT:    s_lshl_b32 s6, s3, 3
-; GFX908-NEXT:    v_mov_b32_e32 v2, s6
-; GFX908-NEXT:    ds_add_rtn_f32 v2, v2, v1
-; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX908-NEXT:  .LBB28_4:
-; GFX908-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX908-NEXT:    s_mov_b64 s[6:7], exec
-; GFX908-NEXT:    v_mbcnt_lo_u32_b32 v1, s6, 0
-; GFX908-NEXT:    v_mbcnt_hi_u32_b32 v1, s7, v1
-; GFX908-NEXT:    v_readfirstlane_b32 s8, v2
-; GFX908-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX908-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX908-NEXT:    s_cbranch_execz .LBB28_6
-; GFX908-NEXT:  ; %bb.5:
-; GFX908-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
-; GFX908-NEXT:    v_cvt_f32_ubyte0_e32 v1, s6
-; GFX908-NEXT:    s_lshl_b32 s3, s3, 4
+; GFX908-NEXT:    s_bcnt1_i32_b64 s2, s[8:9]
+; GFX908-NEXT:    v_cvt_f32_ubyte0_e32 v1, s2
+; GFX908-NEXT:    s_lshl_b32 s2, s5, 4
 ; GFX908-NEXT:    v_mul_f32_e32 v1, 0x42280000, v1
-; GFX908-NEXT:    v_mov_b32_e32 v2, s3
+; GFX908-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX908-NEXT:    ds_add_f32 v2, v1
 ; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX908-NEXT:  .LBB28_6:
-; GFX908-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX908-NEXT:    s_mov_b64 s[4:5], exec
-; GFX908-NEXT:    v_add_f32_e32 v2, s8, v0
+; GFX908-NEXT:  .LBB28_4:
+; GFX908-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX908-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
+; GFX908-NEXT:    v_mul_f32_e32 v0, 0x42280000, v0
 ; GFX908-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX908-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX908-NEXT:    s_mov_b64 s[2:3], exec
+; GFX908-NEXT:    v_add_f32_e32 v2, s10, v0
 ; GFX908-NEXT:    ; implicit-def: $vgpr0
-; GFX908-NEXT:  .LBB28_7: ; %ComputeLoop1
+; GFX908-NEXT:  .LBB28_5: ; %ComputeLoop
 ; GFX908-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT:    s_ff1_i32_b64 s3, s[4:5]
-; GFX908-NEXT:    s_lshl_b64 s[6:7], 1, s3
+; GFX908-NEXT:    s_ff1_i32_b64 s5, s[2:3]
+; GFX908-NEXT:    s_lshl_b64 s[6:7], 1, s5
 ; GFX908-NEXT:    v_readfirstlane_b32 s8, v1
-; GFX908-NEXT:    v_readlane_b32 s9, v2, s3
-; GFX908-NEXT:    s_mov_b32 m0, s3
-; GFX908-NEXT:    s_andn2_b64 s[4:5], s[4:5], s[6:7]
+; GFX908-NEXT:    v_readlane_b32 s9, v2, s5
+; GFX908-NEXT:    s_mov_b32 m0, s5
+; GFX908-NEXT:    s_andn2_b64 s[2:3], s[2:3], s[6:7]
 ; GFX908-NEXT:    v_writelane_b32 v0, s8, m0
-; GFX908-NEXT:    s_cmp_lg_u64 s[4:5], 0
+; GFX908-NEXT:    s_cmp_lg_u64 s[2:3], 0
 ; GFX908-NEXT:    v_add_f32_e32 v1, s9, v1
-; GFX908-NEXT:    s_cbranch_scc1 .LBB28_7
-; GFX908-NEXT:  ; %bb.8: ; %ComputeEnd2
+; GFX908-NEXT:    s_cbranch_scc1 .LBB28_5
+; GFX908-NEXT:  ; %bb.6: ; %ComputeEnd
 ; GFX908-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
 ; GFX908-NEXT:    v_mbcnt_hi_u32_b32 v2, exec_hi, v2
 ; GFX908-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
 ; GFX908-NEXT:    ; implicit-def: $vgpr2
-; GFX908-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX908-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GFX908-NEXT:    s_cbranch_execz .LBB28_10
-; GFX908-NEXT:  ; %bb.9:
-; GFX908-NEXT:    v_mov_b32_e32 v2, s2
+; GFX908-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX908-NEXT:    s_xor_b64 s[2:3], exec, s[2:3]
+; GFX908-NEXT:    s_cbranch_execz .LBB28_8
+; GFX908-NEXT:  ; %bb.7:
+; GFX908-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX908-NEXT:    ds_add_rtn_f32 v2, v2, v1
 ; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX908-NEXT:  .LBB28_10:
-; GFX908-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT:  .LBB28_8:
+; GFX908-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX908-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
 ; GFX908-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX908-NEXT:    v_mov_b32_e32 v1, 0
@@ -7545,89 +7484,78 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
 ;
 ; GFX8-LABEL: local_ds_fadd:
 ; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
 ; GFX8-NEXT:    s_mov_b64 s[2:3], exec
-; GFX8-NEXT:    v_bfrev_b32_e32 v1, 1
-; GFX8-NEXT:    v_mov_b32_e32 v2, 0x42280000
-; GFX8-NEXT:    ; implicit-def: $vgpr0
-; GFX8-NEXT:  .LBB28_1: ; %ComputeLoop
-; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT:    s_ff1_i32_b64 s6, s[2:3]
-; GFX8-NEXT:    s_lshl_b64 s[4:5], 1, s6
-; GFX8-NEXT:    v_readfirstlane_b32 s7, v1
-; GFX8-NEXT:    v_readlane_b32 s8, v2, s6
-; GFX8-NEXT:    s_mov_b32 m0, s6
-; GFX8-NEXT:    s_andn2_b64 s[2:3], s[2:3], s[4:5]
-; GFX8-NEXT:    v_writelane_b32 v0, s7, m0
-; GFX8-NEXT:    s_cmp_lg_u64 s[2:3], 0
-; GFX8-NEXT:    v_add_f32_e32 v1, s8, v1
-; GFX8-NEXT:    s_cbranch_scc1 .LBB28_1
-; GFX8-NEXT:  ; %bb.2: ; %ComputeEnd
-; GFX8-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
-; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX8-NEXT:    ; implicit-def: $vgpr2
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_add_i32 s5, s5, 4
+; GFX8-NEXT:    ; implicit-def: $vgpr1
 ; GFX8-NEXT:    s_mov_b32 m0, -1
+; GFX8-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; GFX8-NEXT:    s_cbranch_execz .LBB28_2
+; GFX8-NEXT:  ; %bb.1:
+; GFX8-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
+; GFX8-NEXT:    s_lshl_b32 s8, s5, 3
+; GFX8-NEXT:    v_cvt_f32_ubyte0_e32 v1, s2
+; GFX8-NEXT:    v_mul_f32_e32 v1, 0x42280000, v1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s8
+; GFX8-NEXT:    ds_add_rtn_f32 v1, v2, v1
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_add_i32 s3, s3, 4
-; GFX8-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX8-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX8-NEXT:  .LBB28_2:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX8-NEXT:    s_mov_b64 s[8:9], exec
+; GFX8-NEXT:    v_readfirstlane_b32 s10, v1
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v1, s8, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v1, s9, v1
+; GFX8-NEXT:    v_cmp_eq_u32_e64 s[2:3], 0, v1
+; GFX8-NEXT:    s_and_saveexec_b64 s[6:7], s[2:3]
 ; GFX8-NEXT:    s_cbranch_execz .LBB28_4
 ; GFX8-NEXT:  ; %bb.3:
-; GFX8-NEXT:    s_lshl_b32 s6, s3, 3
-; GFX8-NEXT:    v_mov_b32_e32 v2, s6
-; GFX8-NEXT:    ds_add_rtn_f32 v2, v2, v1
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:  .LBB28_4:
-; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX8-NEXT:    s_mov_b64 s[6:7], exec
-; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v1, s6, 0
-; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v1, s7, v1
-; GFX8-NEXT:    v_readfirstlane_b32 s8, v2
-; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX8-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX8-NEXT:    s_cbranch_execz .LBB28_6
-; GFX8-NEXT:  ; %bb.5:
-; GFX8-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
-; GFX8-NEXT:    v_cvt_f32_ubyte0_e32 v1, s6
-; GFX8-NEXT:    s_lshl_b32 s3, s3, 4
+; GFX8-NEXT:    s_bcnt1_i32_b64 s2, s[8:9]
+; GFX8-NEXT:    v_cvt_f32_ubyte0_e32 v1, s2
+; GFX8-NEXT:    s_lshl_b32 s2, s5, 4
 ; GFX8-NEXT:    v_mul_f32_e32 v1, 0x42280000, v1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s3
+; GFX8-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX8-NEXT:    ds_add_f32 v2, v1
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:  .LBB28_6:
-; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX8-NEXT:    s_mov_b64 s[4:5], exec
-; GFX8-NEXT:    v_add_f32_e32 v2, s8, v0
+; GFX8-NEXT:  .LBB28_4:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX8-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
+; GFX8-NEXT:    v_mul_f32_e32 v0, 0x42280000, v0
 ; GFX8-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX8-NEXT:    s_mov_b64 s[2:3], exec
+; GFX8-NEXT:    v_add_f32_e32 v2, s10, v0
 ; GFX8-NEXT:    ; implicit-def: $vgpr0
-; GFX8-NEXT:  .LBB28_7: ; %ComputeLoop1
+; GFX8-NEXT:  .LBB28_5: ; %ComputeLoop
 ; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT:    s_ff1_i32_b64 s3, s[4:5]
-; GFX8-NEXT:    s_lshl_b64 s[6:7], 1, s3
+; GFX8-NEXT:    s_ff1_i32_b64 s5, s[2:3]
+; GFX8-NEXT:    s_lshl_b64 s[6:7], 1, s5
 ; GFX8-NEXT:    v_readfirstlane_b32 s8, v1
-; GFX8-NEXT:    v_readlane_b32 s9, v2, s3
-; GFX8-NEXT:    s_mov_b32 m0, s3
-; GFX8-NEXT:    s_andn2_b64 s[4:5], s[4:5], s[6:7]
+; GFX8-NEXT:    v_readlane_b32 s9, v2, s5
+; GFX8-NEXT:    s_mov_b32 m0, s5
+; GFX8-NEXT:    s_andn2_b64 s[2:3], s[2:3], s[6:7]
 ; GFX8-NEXT:    v_writelane_b32 v0, s8, m0
-; GFX8-NEXT:    s_cmp_lg_u64 s[4:5], 0
+; GFX8-NEXT:    s_cmp_lg_u64 s[2:3], 0
 ; GFX8-NEXT:    v_add_f32_e32 v1, s9, v1
-; GFX8-NEXT:    s_cbranch_scc1 .LBB28_7
-; GFX8-NEXT:  ; %bb.8: ; %ComputeEnd2
+; GFX8-NEXT:    s_cbranch_scc1 .LBB28_5
+; GFX8-NEXT:  ; %bb.6: ; %ComputeEnd
 ; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
 ; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v2, exec_hi, v2
 ; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
 ; GFX8-NEXT:    ; implicit-def: $vgpr2
-; GFX8-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX8-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GFX8-NEXT:    s_cbranch_execz .LBB28_10
-; GFX8-NEXT:  ; %bb.9:
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
+; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX8-NEXT:    s_xor_b64 s[2:3], exec, s[2:3]
+; GFX8-NEXT:    s_cbranch_execz .LBB28_8
+; GFX8-NEXT:  ; %bb.7:
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX8-NEXT:    s_mov_b32 m0, -1
 ; GFX8-NEXT:    ds_add_rtn_f32 v2, v2, v1
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:  .LBB28_10:
-; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT:  .LBB28_8:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
 ; GFX8-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX8-NEXT:    v_add_f32_e32 v2, s2, v0
@@ -7639,153 +7567,186 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
 ;
 ; GFX7-LABEL: local_ds_fadd:
 ; GFX7:       ; %bb.0:
-; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
-; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
+; GFX7-NEXT:    s_mov_b64 s[2:3], exec
+; GFX7-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX7-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_lshl_b32 s4, s3, 3
-; GFX7-NEXT:    v_mov_b32_e32 v0, s4
-; GFX7-NEXT:    ds_read_b32 v0, v0 offset:32
-; GFX7-NEXT:    s_add_i32 s3, s3, 4
-; GFX7-NEXT:    s_lshl_b32 s6, s3, 3
-; GFX7-NEXT:    s_mov_b64 s[4:5], 0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s6
-; GFX7-NEXT:  .LBB28_1: ; %atomicrmw.start
+; GFX7-NEXT:    s_add_i32 s5, s5, 4
+; GFX7-NEXT:    ; implicit-def: $vgpr1
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; GFX7-NEXT:    s_cbranch_execz .LBB28_4
+; GFX7-NEXT:  ; %bb.1:
+; GFX7-NEXT:    s_lshl_b32 s8, s5, 3
+; GFX7-NEXT:    v_mov_b32_e32 v2, s8
+; GFX7-NEXT:    ds_read_b32 v1, v2
+; GFX7-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
+; GFX7-NEXT:    v_cvt_f32_ubyte0_e32 v3, s2
+; GFX7-NEXT:    v_mul_f32_e32 v3, 0x42280000, v3
+; GFX7-NEXT:    s_mov_b64 s[8:9], 0
+; GFX7-NEXT:  .LBB28_2: ; %atomicrmw.start
 ; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v2, v0
-; GFX7-NEXT:    v_add_f32_e32 v0, 0x42280000, v2
-; GFX7-NEXT:    ds_cmpst_rtn_b32 v0, v1, v2, v0
+; GFX7-NEXT:    v_mov_b32_e32 v4, v1
+; GFX7-NEXT:    v_add_f32_e32 v1, v4, v3
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v2, v4, v1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
-; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT:    s_cbranch_execnz .LBB28_1
-; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT:    s_mov_b64 s[6:7], exec
-; GFX7-NEXT:    v_mbcnt_lo_u32_b32_e64 v1, s6, 0
-; GFX7-NEXT:    v_mbcnt_hi_u32_b32_e32 v1, s7, v1
-; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX7-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX7-NEXT:    s_cbranch_execz .LBB28_5
-; GFX7-NEXT:  ; %bb.3:
-; GFX7-NEXT:    s_lshl_b32 s3, s3, 4
-; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    v_cmp_eq_u32_e64 s[2:3], v1, v4
+; GFX7-NEXT:    s_or_b64 s[8:9], s[2:3], s[8:9]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; GFX7-NEXT:    s_cbranch_execnz .LBB28_2
+; GFX7-NEXT:  ; %bb.3: ; %Flow18
+; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
+; GFX7-NEXT:  .LBB28_4: ; %Flow19
+; GFX7-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX7-NEXT:    s_mov_b64 s[8:9], exec
+; GFX7-NEXT:    v_readfirstlane_b32 s10, v1
+; GFX7-NEXT:    v_mbcnt_lo_u32_b32_e64 v1, s8, 0
+; GFX7-NEXT:    v_mbcnt_hi_u32_b32_e32 v1, s9, v1
+; GFX7-NEXT:    v_cmp_eq_u32_e64 s[2:3], 0, v1
+; GFX7-NEXT:    s_and_saveexec_b64 s[6:7], s[2:3]
+; GFX7-NEXT:    s_cbranch_execz .LBB28_7
+; GFX7-NEXT:  ; %bb.5:
+; GFX7-NEXT:    s_lshl_b32 s2, s5, 4
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX7-NEXT:    ds_read_b32 v3, v1
-; GFX7-NEXT:    s_bcnt1_i32_b64 s3, s[6:7]
-; GFX7-NEXT:    v_cvt_f32_ubyte0_e32 v2, s3
+; GFX7-NEXT:    s_bcnt1_i32_b64 s2, s[8:9]
+; GFX7-NEXT:    v_cvt_f32_ubyte0_e32 v2, s2
 ; GFX7-NEXT:    v_mul_f32_e32 v2, 0x42280000, v2
-; GFX7-NEXT:    s_mov_b64 s[6:7], 0
-; GFX7-NEXT:  .LBB28_4: ; %atomicrmw.start2
+; GFX7-NEXT:    s_mov_b64 s[8:9], 0
+; GFX7-NEXT:  .LBB28_6: ; %atomicrmw.start2
 ; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    v_add_f32_e32 v4, v3, v2
 ; GFX7-NEXT:    ds_cmpst_rtn_b32 v4, v1, v3, v4
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX7-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7-NEXT:    v_cmp_eq_u32_e64 s[2:3], v4, v3
+; GFX7-NEXT:    s_or_b64 s[8:9], s[2:3], s[8:9]
 ; GFX7-NEXT:    v_mov_b32_e32 v3, v4
-; GFX7-NEXT:    s_andn2_b64 exec, exec, s[6:7]
-; GFX7-NEXT:    s_cbranch_execnz .LBB28_4
-; GFX7-NEXT:  .LBB28_5: ; %Flow17
-; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    ds_read_b32 v1, v2
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; GFX7-NEXT:    s_cbranch_execnz .LBB28_6
+; GFX7-NEXT:  .LBB28_7: ; %Flow17
+; GFX7-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX7-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s4
+; GFX7-NEXT:    v_mul_f32_e32 v2, 0x42280000, v0
+; GFX7-NEXT:    ds_read_b32 v0, v1
+; GFX7-NEXT:    v_bfrev_b32_e32 v3, 1
+; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX7-NEXT:    v_add_f32_e32 v2, s10, v2
 ; GFX7-NEXT:    s_mov_b64 s[2:3], 0
-; GFX7-NEXT:  .LBB28_6: ; %atomicrmw.start8
+; GFX7-NEXT:  .LBB28_8: ; %atomicrmw.start8
 ; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v3, v1
-; GFX7-NEXT:    v_add_f32_e32 v1, v3, v0
-; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v2, v3, v1
+; GFX7-NEXT:    v_mov_b32_e32 v3, v0
+; GFX7-NEXT:    v_add_f32_e32 v0, v3, v2
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v0, v1, v3, v0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v3
 ; GFX7-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
 ; GFX7-NEXT:    s_andn2_b64 exec, exec, s[2:3]
-; GFX7-NEXT:    s_cbranch_execnz .LBB28_6
-; GFX7-NEXT:  ; %bb.7: ; %atomicrmw.end7
+; GFX7-NEXT:    s_cbranch_execnz .LBB28_8
+; GFX7-NEXT:  ; %bb.9: ; %atomicrmw.end7
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
 ; GFX7-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7-NEXT:    s_mov_b32 s2, -1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    buffer_store_dword v1, off, s[0:3], 0
+; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX6-LABEL: local_ds_fadd:
 ; GFX6:       ; %bb.0:
-; GFX6-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
-; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
+; GFX6-NEXT:    s_mov_b64 s[2:3], exec
+; GFX6-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX6-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    s_lshl_b32 s4, s3, 3
-; GFX6-NEXT:    s_add_i32 s4, s4, 32
-; GFX6-NEXT:    v_mov_b32_e32 v0, s4
-; GFX6-NEXT:    ds_read_b32 v0, v0
-; GFX6-NEXT:    s_add_i32 s3, s3, 4
-; GFX6-NEXT:    s_lshl_b32 s6, s3, 3
-; GFX6-NEXT:    s_mov_b64 s[4:5], 0
-; GFX6-NEXT:    v_mov_b32_e32 v1, s6
-; GFX6-NEXT:  .LBB28_1: ; %atomicrmw.start
+; GFX6-NEXT:    s_add_i32 s5, s5, 4
+; GFX6-NEXT:    ; implicit-def: $vgpr1
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; GFX6-NEXT:    s_cbranch_execz .LBB28_4
+; GFX6-NEXT:  ; %bb.1:
+; GFX6-NEXT:    s_lshl_b32 s8, s5, 3
+; GFX6-NEXT:    v_mov_b32_e32 v2, s8
+; GFX6-NEXT:    ds_read_b32 v1, v2
+; GFX6-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
+; GFX6-NEXT:    v_cvt_f32_ubyte0_e32 v3, s2
+; GFX6-NEXT:    v_mul_f32_e32 v3, 0x42280000, v3
+; GFX6-NEXT:    s_mov_b64 s[8:9], 0
+; GFX6-NEXT:  .LBB28_2: ; %atomicrmw.start
 ; GFX6-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    v_mov_b32_e32 v2, v0
-; GFX6-NEXT:    v_add_f32_e32 v0, 0x42280000, v2
-; GFX6-NEXT:    ds_cmpst_rtn_b32 v0, v1, v2, v0
+; GFX6-NEXT:    v_mov_b32_e32 v4, v1
+; GFX6-NEXT:    v_add_f32_e32 v1, v4, v3
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v2, v4, v1
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
-; GFX6-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX6-NEXT:    s_cbranch_execnz .LBB28_1
-; GFX6-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX6-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX6-NEXT:    s_mov_b64 s[6:7], exec
-; GFX6-NEXT:    v_mbcnt_lo_u32_b32_e64 v1, s6, 0
-; GFX6-NEXT:    v_mbcnt_hi_u32_b32_e32 v1, s7, v1
-; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX6-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX6-NEXT:    s_cbranch_execz .LBB28_5
-; GFX6-NEXT:  ; %bb.3:
-; GFX6-NEXT:    s_lshl_b32 s3, s3, 4
-; GFX6-NEXT:    v_mov_b32_e32 v1, s3
+; GFX6-NEXT:    v_cmp_eq_u32_e64 s[2:3], v1, v4
+; GFX6-NEXT:    s_or_b64 s[8:9], s[2:3], s[8:9]
+; GFX6-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; GFX6-NEXT:    s_cbranch_execnz .LBB28_2
+; GFX6-NEXT:  ; %bb.3: ; %Flow16
+; GFX6-NEXT:    s_or_b64 exec, exec, s[8:9]
+; GFX6-NEXT:  .LBB28_4: ; %Flow17
+; GFX6-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX6-NEXT:    s_mov_b64 s[8:9], exec
+; GFX6-NEXT:    v_readfirstlane_b32 s10, v1
+; GFX6-NEXT:    v_mbcnt_lo_u32_b32_e64 v1, s8, 0
+; GFX6-NEXT:    v_mbcnt_hi_u32_b32_e32 v1, s9, v1
+; GFX6-NEXT:    v_cmp_eq_u32_e64 s[2:3], 0, v1
+; GFX6-NEXT:    s_and_saveexec_b64 s[6:7], s[2:3]
+; GFX6-NEXT:    s_cbranch_execz .LBB28_7
+; GFX6-NEXT:  ; %bb.5:
+; GFX6-NEXT:    s_lshl_b32 s2, s5, 4
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX6-NEXT:    ds_read_b32 v3, v1
-; GFX6-NEXT:    s_bcnt1_i32_b64 s3, s[6:7]
-; GFX6-NEXT:    v_cvt_f32_ubyte0_e32 v2, s3
+; GFX6-NEXT:    s_bcnt1_i32_b64 s2, s[8:9]
+; GFX6-NEXT:    v_cvt_f32_ubyte0_e32 v2, s2
 ; GFX6-NEXT:    v_mul_f32_e32 v2, 0x42280000, v2
-; GFX6-NEXT:    s_mov_b64 s[6:7], 0
-; GFX6-NEXT:  .LBB28_4: ; %atomicrmw.start2
+; GFX6-NEXT:    s_mov_b64 s[8:9], 0
+; GFX6-NEXT:  .LBB28_6: ; %atomicrmw.start2
 ; GFX6-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX6-NEXT:    v_add_f32_e32 v4, v3, v2
 ; GFX6-NEXT:    ds_cmpst_rtn_b32 v4, v1, v3, v4
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX6-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GFX6-NEXT:    v_cmp_eq_u32_e64 s[2:3], v4, v3
+; GFX6-NEXT:    s_or_b64 s[8:9], s[2:3], s[8:9]
 ; GFX6-NEXT:    v_mov_b32_e32 v3, v4
-; GFX6-NEXT:    s_andn2_b64 exec, exec, s[6:7]
-; GFX6-NEXT:    s_cbranch_execnz .LBB28_4
-; GFX6-NEXT:  .LBB28_5: ; %Flow15
-; GFX6-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX6-NEXT:    v_mov_b32_e32 v2, s2
-; GFX6-NEXT:    ds_read_b32 v1, v2
+; GFX6-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; GFX6-NEXT:    s_cbranch_execnz .LBB28_6
+; GFX6-NEXT:  .LBB28_7: ; %Flow15
+; GFX6-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX6-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s4
+; GFX6-NEXT:    v_mul_f32_e32 v2, 0x42280000, v0
+; GFX6-NEXT:    ds_read_b32 v0, v1
+; GFX6-NEXT:    v_bfrev_b32_e32 v3, 1
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX6-NEXT:    v_add_f32_e32 v2, s10, v2
 ; GFX6-NEXT:    s_mov_b64 s[2:3], 0
-; GFX6-NEXT:  .LBB28_6: ; %atomicrmw.start8
+; GFX6-NEXT:  .LBB28_8: ; %atomicrmw.start8
 ; GFX6-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    v_mov_b32_e32 v3, v1
-; GFX6-NEXT:    v_add_f32_e32 v1, v3, v0
-; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v2, v3, v1
+; GFX6-NEXT:    v_mov_b32_e32 v3, v0
+; GFX6-NEXT:    v_add_f32_e32 v0, v3, v2
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v0, v1, v3, v0
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v3
 ; GFX6-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
 ; GFX6-NEXT:    s_andn2_b64 exec, exec, s[2:3]
-; GFX6-NEXT:    s_cbranch_execnz .LBB28_6
-; GFX6-NEXT:  ; %bb.7: ; %atomicrmw.end7
+; GFX6-NEXT:    s_cbranch_execnz .LBB28_8
+; GFX6-NEXT:  ; %bb.9: ; %atomicrmw.end7
 ; GFX6-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX6-NEXT:    s_mov_b32 s2, -1
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    buffer_store_dword v1, off, s[0:3], 0
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX6-NEXT:    s_endpgm
   %idx.add = add nuw i32 %idx, 4
   %shl0 = shl i32 %idx.add, 3
@@ -7802,86 +7763,78 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
 define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrspace(3) %ptrf, i32 %idx) {
 ; GFX12-LABEL: local_ds_fadd_one_as:
 ; GFX12:       ; %bb.0:
-; GFX12-NEXT:    v_mov_b32_e32 v1, 0x42280000
-; GFX12-NEXT:    s_mov_b32 s2, exec_lo
-; GFX12-NEXT:    s_brev_b32 s4, 1
-; GFX12-NEXT:    ; implicit-def: $vgpr0
-; GFX12-NEXT:  .LBB29_1: ; %ComputeLoop
-; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT:    s_ctz_i32_b32 s3, s2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT:    v_readlane_b32 s5, v1, s3
-; GFX12-NEXT:    s_lshl_b32 s6, 1, s3
-; GFX12-NEXT:    v_writelane_b32 v0, s4, s3
-; GFX12-NEXT:    s_and_not1_b32 s2, s2, s6
-; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT:    s_cmp_lg_u32 s2, 0
-; GFX12-NEXT:    s_add_f32 s4, s4, s5
-; GFX12-NEXT:    s_cbranch_scc1 .LBB29_1
-; GFX12-NEXT:  ; %bb.2: ; %ComputeEnd
-; GFX12-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
-; GFX12-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX12-NEXT:    s_load_b64 s[4:5], s[0:1], 0x8
+; GFX12-NEXT:    s_mov_b32 s6, exec_lo
 ; GFX12-NEXT:    ; implicit-def: $vgpr1
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_add_co_i32 s3, s3, 4
-; GFX12-NEXT:    s_and_saveexec_b32 s5, vcc_lo
-; GFX12-NEXT:    s_xor_b32 s5, exec_lo, s5
-; GFX12-NEXT:    s_cbranch_execz .LBB29_4
-; GFX12-NEXT:  ; %bb.3:
-; GFX12-NEXT:    s_lshl_b32 s6, s3, 3
+; GFX12-NEXT:    s_add_co_i32 s3, s5, 4
+; GFX12-NEXT:    s_and_saveexec_b32 s2, vcc_lo
+; GFX12-NEXT:    s_cbranch_execz .LBB29_2
+; GFX12-NEXT:  ; %bb.1:
+; GFX12-NEXT:    s_bcnt1_i32_b32 s5, s6
 ; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v1, s6
-; GFX12-NEXT:    ds_add_rtn_f32 v1, v1, v2
-; GFX12-NEXT:  .LBB29_4:
-; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s5
+; GFX12-NEXT:    v_cvt_f32_ubyte0_e32 v1, s5
+; GFX12-NEXT:    s_lshl_b32 s5, s3, 3
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT:    v_dual_mov_b32 v2, s5 :: v_dual_mul_f32 v1, 0x42280000, v1
+; GFX12-NEXT:    ds_add_rtn_f32 v1, v2, v1
+; GFX12-NEXT:  .LBB29_2:
+; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s2
 ; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT:    s_mov_b32 s6, exec_lo
+; GFX12-NEXT:    s_mov_b32 s7, exec_lo
 ; GFX12-NEXT:    s_wait_dscnt 0x0
 ; GFX12-NEXT:    v_readfirstlane_b32 s5, v1
-; GFX12-NEXT:    v_mbcnt_lo_u32_b32 v2, s6, 0
-; GFX12-NEXT:    s_mov_b32 s4, exec_lo
+; GFX12-NEXT:    v_mbcnt_lo_u32_b32 v2, s7, 0
+; GFX12-NEXT:    s_mov_b32 s6, exec_lo
 ; GFX12-NEXT:    v_cmpx_eq_u32_e32 0, v2
-; GFX12-NEXT:    s_cbranch_execz .LBB29_6
-; GFX12-NEXT:  ; %bb.5:
-; GFX12-NEXT:    s_bcnt1_i32_b32 s6, s6
-; GFX12-NEXT:    s_lshl_b32 s3, s3, 4
-; GFX12-NEXT:    v_cvt_f32_ubyte0_e32 v1, s6
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    v_dual_mov_b32 v2, s3 :: v_dual_mul_f32 v1, 0x42280000, v1
+; GFX12-NEXT:    s_cbranch_execz .LBB29_4
+; GFX12-NEXT:  ; %bb.3:
+; GFX12-NEXT:    s_bcnt1_i32_b32 s2, s7
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    v_cvt_f32_ubyte0_e32 v1, s2
+; GFX12-NEXT:    s_lshl_b32 s2, s3, 4
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mul_f32 v1, 0x42280000, v1
 ; GFX12-NEXT:    ds_add_f32 v2, v1
-; GFX12-NEXT:  .LBB29_6:
-; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:  .LBB29_4:
+; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s6
+; GFX12-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
+; GFX12-NEXT:    s_mov_b32 s3, exec_lo
+; GFX12-NEXT:    s_brev_b32 s2, 1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_mul_f32_e32 v0, 0x42280000, v0
+; GFX12-NEXT:    v_cndmask_b32_e64 v0, v0, 0x80000000, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_add_f32_e32 v1, s5, v0
-; GFX12-NEXT:    s_mov_b32 s4, exec_lo
-; GFX12-NEXT:    s_brev_b32 s3, 1
 ; GFX12-NEXT:    ; implicit-def: $vgpr0
-; GFX12-NEXT:  .LBB29_7: ; %ComputeLoop1
+; GFX12-NEXT:  .LBB29_5: ; %ComputeLoop
 ; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT:    s_ctz_i32_b32 s5, s4
+; GFX12-NEXT:    s_ctz_i32_b32 s5, s3
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
 ; GFX12-NEXT:    v_readlane_b32 s6, v1, s5
 ; GFX12-NEXT:    s_lshl_b32 s7, 1, s5
-; GFX12-NEXT:    v_writelane_b32 v0, s3, s5
-; GFX12-NEXT:    s_and_not1_b32 s4, s4, s7
+; GFX12-NEXT:    v_writelane_b32 v0, s2, s5
+; GFX12-NEXT:    s_and_not1_b32 s3, s3, s7
 ; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT:    s_cmp_lg_u32 s4, 0
-; GFX12-NEXT:    s_add_f32 s3, s3, s6
-; GFX12-NEXT:    s_cbranch_scc1 .LBB29_7
-; GFX12-NEXT:  ; %bb.8: ; %ComputeEnd2
+; GFX12-NEXT:    s_cmp_lg_u32 s3, 0
+; GFX12-NEXT:    s_add_f32 s2, s2, s6
+; GFX12-NEXT:    s_cbranch_scc1 .LBB29_5
+; GFX12-NEXT:  ; %bb.6: ; %ComputeEnd
 ; GFX12-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
 ; GFX12-NEXT:    ; implicit-def: $vgpr1
-; GFX12-NEXT:    s_and_saveexec_b32 s4, vcc_lo
-; GFX12-NEXT:    s_xor_b32 s4, exec_lo, s4
-; GFX12-NEXT:    s_cbranch_execz .LBB29_10
-; GFX12-NEXT:  ; %bb.9:
-; GFX12-NEXT:    v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s3
+; GFX12-NEXT:    s_and_saveexec_b32 s3, vcc_lo
+; GFX12-NEXT:    s_xor_b32 s3, exec_lo, s3
+; GFX12-NEXT:    s_cbranch_execz .LBB29_8
+; GFX12-NEXT:  ; %bb.7:
+; GFX12-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_mov_b32 v2, s2
 ; GFX12-NEXT:    ds_add_rtn_f32 v1, v1, v2
-; GFX12-NEXT:  .LBB29_10:
-; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:  .LBB29_8:
+; GFX12-NEXT:    s_or_b32 exec_lo, exec_lo, s3
 ; GFX12-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX12-NEXT:    s_wait_dscnt 0x0
 ; GFX12-NEXT:    v_readfirstlane_b32 s2, v1
@@ -7895,85 +7848,74 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
 ;
 ; GFX940-LABEL: local_ds_fadd_one_as:
 ; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
 ; GFX940-NEXT:    s_mov_b64 s[2:3], exec
-; GFX940-NEXT:    v_bfrev_b32_e32 v1, 1
-; GFX940-NEXT:    v_mov_b32_e32 v2, 0x42280000
-; GFX940-NEXT:    ; implicit-def: $vgpr0
-; GFX940-NEXT:  .LBB29_1: ; %ComputeLoop
-; GFX940-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX940-NEXT:    s_ff1_i32_b64 s6, s[2:3]
-; GFX940-NEXT:    s_lshl_b64 s[4:5], 1, s6
-; GFX940-NEXT:    v_readfirstlane_b32 s7, v1
-; GFX940-NEXT:    v_readlane_b32 s8, v2, s6
-; GFX940-NEXT:    s_mov_b32 m0, s6
-; GFX940-NEXT:    s_andn2_b64 s[2:3], s[2:3], s[4:5]
-; GFX940-NEXT:    v_writelane_b32 v0, s7, m0
-; GFX940-NEXT:    s_cmp_lg_u64 s[2:3], 0
-; GFX940-NEXT:    v_add_f32_e32 v1, s8, v1
-; GFX940-NEXT:    s_cbranch_scc1 .LBB29_1
-; GFX940-NEXT:  ; %bb.2: ; %ComputeEnd
-; GFX940-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
-; GFX940-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX940-NEXT:    v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX940-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
-; GFX940-NEXT:    ; implicit-def: $vgpr2
+; GFX940-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX940-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX940-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NEXT:    s_add_i32 s3, s3, 4
-; GFX940-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX940-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX940-NEXT:    s_add_i32 s5, s5, 4
+; GFX940-NEXT:    ; implicit-def: $vgpr1
+; GFX940-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; GFX940-NEXT:    s_cbranch_execz .LBB29_2
+; GFX940-NEXT:  ; %bb.1:
+; GFX940-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
+; GFX940-NEXT:    s_lshl_b32 s8, s5, 3
+; GFX940-NEXT:    v_cvt_f32_ubyte0_e32 v1, s2
+; GFX940-NEXT:    v_mul_f32_e32 v1, 0x42280000, v1
+; GFX940-NEXT:    v_mov_b32_e32 v2, s8
+; GFX940-NEXT:    ds_add_rtn_f32 v1, v2, v1
+; GFX940-NEXT:  .LBB29_2:
+; GFX940-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX940-NEXT:    s_mov_b64 s[8:9], exec
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_readfirstlane_b32 s10, v1
+; GFX940-NEXT:    v_mbcnt_lo_u32_b32 v1, s8, 0
+; GFX940-NEXT:    v_mbcnt_hi_u32_b32 v1, s9, v1
+; GFX940-NEXT:    v_cmp_eq_u32_e64 s[2:3], 0, v1
+; GFX940-NEXT:    s_and_saveexec_b64 s[6:7], s[2:3]
 ; GFX940-NEXT:    s_cbranch_execz .LBB29_4
 ; GFX940-NEXT:  ; %bb.3:
-; GFX940-NEXT:    s_lshl_b32 s6, s3, 3
-; GFX940-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-NEXT:    ds_add_rtn_f32 v2, v2, v1
-; GFX940-NEXT:  .LBB29_4:
-; GFX940-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX940-NEXT:    s_mov_b64 s[6:7], exec
-; GFX940-NEXT:    v_mbcnt_lo_u32_b32 v1, s6, 0
-; GFX940-NEXT:    v_mbcnt_hi_u32_b32 v1, s7, v1
-; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NEXT:    v_readfirstlane_b32 s8, v2
-; GFX940-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX940-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX940-NEXT:    s_cbranch_execz .LBB29_6
-; GFX940-NEXT:  ; %bb.5:
-; GFX940-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
-; GFX940-NEXT:    v_cvt_f32_ubyte0_e32 v1, s6
-; GFX940-NEXT:    s_lshl_b32 s3, s3, 4
+; GFX940-NEXT:    s_bcnt1_i32_b64 s2, s[8:9]
+; GFX940-NEXT:    v_cvt_f32_ubyte0_e32 v1, s2
+; GFX940-NEXT:    s_lshl_b32 s2, s5, 4
 ; GFX940-NEXT:    v_mul_f32_e32 v1, 0x42280000, v1
-; GFX940-NEXT:    v_mov_b32_e32 v2, s3
+; GFX940-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX940-NEXT:    ds_add_f32 v2, v1
-; GFX940-NEXT:  .LBB29_6:
-; GFX940-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX940-NEXT:    s_mov_b64 s[4:5], exec
-; GFX940-NEXT:    v_add_f32_e32 v2, s8, v0
+; GFX940-NEXT:  .LBB29_4:
+; GFX940-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX940-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
+; GFX940-NEXT:    v_mul_f32_e32 v0, 0x42280000, v0
 ; GFX940-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX940-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX940-NEXT:    s_mov_b64 s[2:3], exec
+; GFX940-NEXT:    v_add_f32_e32 v2, s10, v0
 ; GFX940-NEXT:    ; implicit-def: $vgpr0
-; GFX940-NEXT:  .LBB29_7: ; %ComputeLoop1
+; GFX940-NEXT:  .LBB29_5: ; %ComputeLoop
 ; GFX940-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX940-NEXT:    s_ff1_i32_b64 s3, s[4:5]
-; GFX940-NEXT:    s_lshl_b64 s[6:7], 1, s3
+; GFX940-NEXT:    s_ff1_i32_b64 s5, s[2:3]
+; GFX940-NEXT:    s_lshl_b64 s[6:7], 1, s5
 ; GFX940-NEXT:    v_readfirstlane_b32 s8, v1
-; GFX940-NEXT:    v_readlane_b32 s9, v2, s3
-; GFX940-NEXT:    s_mov_b32 m0, s3
-; GFX940-NEXT:    s_andn2_b64 s[4:5], s[4:5], s[6:7]
+; GFX940-NEXT:    v_readlane_b32 s9, v2, s5
+; GFX940-NEXT:    s_mov_b32 m0, s5
+; GFX940-NEXT:    s_andn2_b64 s[2:3], s[2:3], s[6:7]
 ; GFX940-NEXT:    v_writelane_b32 v0, s8, m0
-; GFX940-NEXT:    s_cmp_lg_u64 s[4:5], 0
+; GFX940-NEXT:    s_cmp_lg_u64 s[2:3], 0
 ; GFX940-NEXT:    v_add_f32_e32 v1, s9, v1
-; GFX940-NEXT:    s_cbranch_scc1 .LBB29_7
-; GFX940-NEXT:  ; %bb.8: ; %ComputeEnd2
+; GFX940-NEXT:    s_cbranch_scc1 .LBB29_5
+; GFX940-NEXT:  ; %bb.6: ; %ComputeEnd
 ; GFX940-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
 ; GFX940-NEXT:    v_mbcnt_hi_u32_b32 v2, exec_hi, v2
 ; GFX940-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
 ; GFX940-NEXT:    ; implicit-def: $vgpr2
-; GFX940-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX940-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GFX940-NEXT:    s_cbranch_execz .LBB29_10
-; GFX940-NEXT:  ; %bb.9:
-; GFX940-NEXT:    v_mov_b32_e32 v2, s2
+; GFX940-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX940-NEXT:    s_xor_b64 s[2:3], exec, s[2:3]
+; GFX940-NEXT:    s_cbranch_execz .LBB29_8
+; GFX940-NEXT:  ; %bb.7:
+; GFX940-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX940-NEXT:    ds_add_rtn_f32 v2, v2, v1
-; GFX940-NEXT:  .LBB29_10:
-; GFX940-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX940-NEXT:  .LBB29_8:
+; GFX940-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX940-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
 ; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX940-NEXT:    v_readfirstlane_b32 s2, v2
@@ -7985,90 +7927,80 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
 ;
 ; GFX11-LABEL: local_ds_fadd_one_as:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    v_bfrev_b32_e32 v1, 1
-; GFX11-NEXT:    v_mov_b32_e32 v2, 0x42280000
-; GFX11-NEXT:    s_mov_b32 s2, exec_lo
-; GFX11-NEXT:    ; implicit-def: $vgpr0
-; GFX11-NEXT:  .LBB29_1: ; %ComputeLoop
-; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    s_load_b64 s[4:5], s[0:1], 0x8
+; GFX11-NEXT:    s_mov_b32 s6, exec_lo
+; GFX11-NEXT:    ; implicit-def: $vgpr1
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    s_ctz_i32_b32 s3, s2
-; GFX11-NEXT:    v_readfirstlane_b32 s4, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT:    v_readlane_b32 s5, v2, s3
-; GFX11-NEXT:    s_lshl_b32 s6, 1, s3
-; GFX11-NEXT:    s_and_not1_b32 s2, s2, s6
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_writelane_b32 v0, s4, s3
-; GFX11-NEXT:    v_add_f32_e32 v1, s5, v1
-; GFX11-NEXT:    s_cmp_lg_u32 s2, 0
-; GFX11-NEXT:    s_cbranch_scc1 .LBB29_1
-; GFX11-NEXT:  ; %bb.2: ; %ComputeEnd
-; GFX11-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
-; GFX11-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v2
-; GFX11-NEXT:    ; implicit-def: $vgpr2
+; GFX11-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_add_i32 s3, s3, 4
-; GFX11-NEXT:    s_and_saveexec_b32 s4, vcc_lo
-; GFX11-NEXT:    s_xor_b32 s4, exec_lo, s4
-; GFX11-NEXT:    s_cbranch_execz .LBB29_4
-; GFX11-NEXT:  ; %bb.3:
-; GFX11-NEXT:    s_lshl_b32 s5, s3, 3
+; GFX11-NEXT:    s_add_i32 s3, s5, 4
+; GFX11-NEXT:    s_and_saveexec_b32 s2, vcc_lo
+; GFX11-NEXT:    s_cbranch_execz .LBB29_2
+; GFX11-NEXT:  ; %bb.1:
+; GFX11-NEXT:    s_bcnt1_i32_b32 s5, s6
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT:    v_mov_b32_e32 v2, s5
-; GFX11-NEXT:    ds_add_rtn_f32 v2, v2, v1
-; GFX11-NEXT:  .LBB29_4:
-; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    v_cvt_f32_ubyte0_e32 v1, s5
+; GFX11-NEXT:    s_lshl_b32 s5, s3, 3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v2, s5 :: v_dual_mul_f32 v1, 0x42280000, v1
+; GFX11-NEXT:    ds_add_rtn_f32 v1, v2, v1
+; GFX11-NEXT:  .LBB29_2:
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s2
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    s_mov_b32 s6, exec_lo
+; GFX11-NEXT:    s_mov_b32 s7, exec_lo
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    v_readfirstlane_b32 s4, v2
-; GFX11-NEXT:    v_mbcnt_lo_u32_b32 v1, s6, 0
-; GFX11-NEXT:    s_mov_b32 s5, exec_lo
-; GFX11-NEXT:    v_cmpx_eq_u32_e32 0, v1
-; GFX11-NEXT:    s_cbranch_execz .LBB29_6
-; GFX11-NEXT:  ; %bb.5:
-; GFX11-NEXT:    s_bcnt1_i32_b32 s6, s6
-; GFX11-NEXT:    s_lshl_b32 s3, s3, 4
-; GFX11-NEXT:    v_cvt_f32_ubyte0_e32 v1, s6
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_dual_mov_b32 v2, s3 :: v_dual_mul_f32 v1, 0x42280000, v1
+; GFX11-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX11-NEXT:    v_mbcnt_lo_u32_b32 v2, s7, 0
+; GFX11-NEXT:    s_mov_b32 s6, exec_lo
+; GFX11-NEXT:    v_cmpx_eq_u32_e32 0, v2
+; GFX11-NEXT:    s_cbranch_execz .LBB29_4
+; GFX11-NEXT:  ; %bb.3:
+; GFX11-NEXT:    s_bcnt1_i32_b32 s2, s7
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_cvt_f32_ubyte0_e32 v1, s2
+; GFX11-NEXT:    s_lshl_b32 s2, s3, 4
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mul_f32 v1, 0x42280000, v1
 ; GFX11-NEXT:    ds_add_f32 v2, v1
-; GFX11-NEXT:  .LBB29_6:
-; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT:    v_add_f32_e32 v2, s4, v0
+; GFX11-NEXT:  .LBB29_4:
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s6
+; GFX11-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
 ; GFX11-NEXT:    v_bfrev_b32_e32 v1, 1
-; GFX11-NEXT:    s_mov_b32 s3, exec_lo
+; GFX11-NEXT:    s_mov_b32 s2, exec_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_mul_f32_e32 v0, 0x42280000, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, 0x80000000, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_add_f32_e32 v2, s5, v0
 ; GFX11-NEXT:    ; implicit-def: $vgpr0
-; GFX11-NEXT:  .LBB29_7: ; %ComputeLoop1
+; GFX11-NEXT:  .LBB29_5: ; %ComputeLoop
 ; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    s_ctz_i32_b32 s4, s3
+; GFX11-NEXT:    s_ctz_i32_b32 s3, s2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_readfirstlane_b32 s5, v1
-; GFX11-NEXT:    v_readlane_b32 s6, v2, s4
-; GFX11-NEXT:    s_lshl_b32 s7, 1, s4
+; GFX11-NEXT:    v_readlane_b32 s6, v2, s3
+; GFX11-NEXT:    s_lshl_b32 s7, 1, s3
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    s_and_not1_b32 s3, s3, s7
-; GFX11-NEXT:    v_writelane_b32 v0, s5, s4
+; GFX11-NEXT:    s_and_not1_b32 s2, s2, s7
+; GFX11-NEXT:    v_writelane_b32 v0, s5, s3
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-NEXT:    v_add_f32_e32 v1, s6, v1
-; GFX11-NEXT:    s_cmp_lg_u32 s3, 0
-; GFX11-NEXT:    s_cbranch_scc1 .LBB29_7
-; GFX11-NEXT:  ; %bb.8: ; %ComputeEnd2
+; GFX11-NEXT:    s_cmp_lg_u32 s2, 0
+; GFX11-NEXT:    s_cbranch_scc1 .LBB29_5
+; GFX11-NEXT:  ; %bb.6: ; %ComputeEnd
 ; GFX11-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v2
 ; GFX11-NEXT:    ; implicit-def: $vgpr2
-; GFX11-NEXT:    s_and_saveexec_b32 s3, vcc_lo
-; GFX11-NEXT:    s_xor_b32 s3, exec_lo, s3
-; GFX11-NEXT:    s_cbranch_execz .LBB29_10
-; GFX11-NEXT:  ; %bb.9:
-; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    s_and_saveexec_b32 s2, vcc_lo
+; GFX11-NEXT:    s_xor_b32 s2, exec_lo, s2
+; GFX11-NEXT:    s_cbranch_execz .LBB29_8
+; GFX11-NEXT:  ; %bb.7:
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX11-NEXT:    ds_add_rtn_f32 v2, v2, v1
-; GFX11-NEXT:  .LBB29_10:
-; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s3
+; GFX11-NEXT:  .LBB29_8:
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s2
 ; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    v_readfirstlane_b32 s2, v2
@@ -8081,83 +8013,73 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
 ;
 ; GFX10-LABEL: local_ds_fadd_one_as:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_bfrev_b32_e32 v1, 1
-; GFX10-NEXT:    v_mov_b32_e32 v2, 0x42280000
-; GFX10-NEXT:    s_mov_b32 s2, exec_lo
-; GFX10-NEXT:    ; implicit-def: $vgpr0
-; GFX10-NEXT:  .LBB29_1: ; %ComputeLoop
-; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT:    s_ff1_i32_b32 s3, s2
-; GFX10-NEXT:    v_readfirstlane_b32 s4, v1
-; GFX10-NEXT:    v_readlane_b32 s5, v2, s3
-; GFX10-NEXT:    s_lshl_b32 s6, 1, s3
-; GFX10-NEXT:    s_andn2_b32 s2, s2, s6
-; GFX10-NEXT:    v_writelane_b32 v0, s4, s3
-; GFX10-NEXT:    v_add_f32_e32 v1, s5, v1
-; GFX10-NEXT:    s_cmp_lg_u32 s2, 0
-; GFX10-NEXT:    s_cbranch_scc1 .LBB29_1
-; GFX10-NEXT:  ; %bb.2: ; %ComputeEnd
-; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
-; GFX10-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    ; implicit-def: $vgpr2
+; GFX10-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
+; GFX10-NEXT:    s_mov_b32 s6, exec_lo
+; GFX10-NEXT:    ; implicit-def: $vgpr1
+; GFX10-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    s_add_i32 s3, s3, 4
-; GFX10-NEXT:    s_and_saveexec_b32 s4, vcc_lo
-; GFX10-NEXT:    s_xor_b32 s4, exec_lo, s4
-; GFX10-NEXT:    s_cbranch_execz .LBB29_4
-; GFX10-NEXT:  ; %bb.3:
+; GFX10-NEXT:    s_add_i32 s3, s5, 4
+; GFX10-NEXT:    s_and_saveexec_b32 s2, vcc_lo
+; GFX10-NEXT:    s_cbranch_execz .LBB29_2
+; GFX10-NEXT:  ; %bb.1:
+; GFX10-NEXT:    s_bcnt1_i32_b32 s5, s6
+; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v1, s5
 ; GFX10-NEXT:    s_lshl_b32 s5, s3, 3
 ; GFX10-NEXT:    v_mov_b32_e32 v2, s5
-; GFX10-NEXT:    ds_add_rtn_f32 v2, v2, v1
-; GFX10-NEXT:  .LBB29_4:
+; GFX10-NEXT:    v_mul_f32_e32 v1, 0x42280000, v1
+; GFX10-NEXT:    ds_add_rtn_f32 v1, v2, v1
+; GFX10-NEXT:  .LBB29_2:
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT:    s_mov_b32 s6, exec_lo
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s2
+; GFX10-NEXT:    s_mov_b32 s7, exec_lo
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_readfirstlane_b32 s4, v2
-; GFX10-NEXT:    v_mbcnt_lo_u32_b32 v1, s6, 0
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX10-NEXT:    s_and_saveexec_b32 s5, vcc_lo
-; GFX10-NEXT:    s_cbranch_execz .LBB29_6
-; GFX10-NEXT:  ; %bb.5:
-; GFX10-NEXT:    s_bcnt1_i32_b32 s6, s6
-; GFX10-NEXT:    s_lshl_b32 s3, s3, 4
-; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v1, s6
-; GFX10-NEXT:    v_mov_b32_e32 v2, s3
+; GFX10-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX10-NEXT:    v_mbcnt_lo_u32_b32 v2, s7, 0
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s2, 0, v2
+; GFX10-NEXT:    s_and_saveexec_b32 s6, s2
+; GFX10-NEXT:    s_cbranch_execz .LBB29_4
+; GFX10-NEXT:  ; %bb.3:
+; GFX10-NEXT:    s_bcnt1_i32_b32 s2, s7
+; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v1, s2
+; GFX10-NEXT:    s_lshl_b32 s2, s3, 4
+; GFX10-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX10-NEXT:    v_mul_f32_e32 v1, 0x42280000, v1
 ; GFX10-NEXT:    ds_add_f32 v2, v1
-; GFX10-NEXT:  .LBB29_6:
+; GFX10-NEXT:  .LBB29_4:
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT:    v_add_f32_e32 v2, s4, v0
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s6
+; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
 ; GFX10-NEXT:    v_bfrev_b32_e32 v1, 1
-; GFX10-NEXT:    s_mov_b32 s3, exec_lo
+; GFX10-NEXT:    s_mov_b32 s2, exec_lo
+; GFX10-NEXT:    v_mul_f32_e32 v0, 0x42280000, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, 0x80000000, vcc_lo
+; GFX10-NEXT:    v_add_f32_e32 v2, s5, v0
 ; GFX10-NEXT:    ; implicit-def: $vgpr0
-; GFX10-NEXT:  .LBB29_7: ; %ComputeLoop1
+; GFX10-NEXT:  .LBB29_5: ; %ComputeLoop
 ; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT:    s_ff1_i32_b32 s4, s3
+; GFX10-NEXT:    s_ff1_i32_b32 s3, s2
 ; GFX10-NEXT:    v_readfirstlane_b32 s5, v1
-; GFX10-NEXT:    v_readlane_b32 s6, v2, s4
-; GFX10-NEXT:    s_lshl_b32 s7, 1, s4
-; GFX10-NEXT:    s_andn2_b32 s3, s3, s7
-; GFX10-NEXT:    v_writelane_b32 v0, s5, s4
+; GFX10-NEXT:    v_readlane_b32 s6, v2, s3
+; GFX10-NEXT:    s_lshl_b32 s7, 1, s3
+; GFX10-NEXT:    s_andn2_b32 s2, s2, s7
+; GFX10-NEXT:    v_writelane_b32 v0, s5, s3
 ; GFX10-NEXT:    v_add_f32_e32 v1, s6, v1
-; GFX10-NEXT:    s_cmp_lg_u32 s3, 0
-; GFX10-NEXT:    s_cbranch_scc1 .LBB29_7
-; GFX10-NEXT:  ; %bb.8: ; %ComputeEnd2
+; GFX10-NEXT:    s_cmp_lg_u32 s2, 0
+; GFX10-NEXT:    s_cbranch_scc1 .LBB29_5
+; GFX10-NEXT:  ; %bb.6: ; %ComputeEnd
 ; GFX10-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v2
 ; GFX10-NEXT:    ; implicit-def: $vgpr2
-; GFX10-NEXT:    s_and_saveexec_b32 s3, vcc_lo
-; GFX10-NEXT:    s_xor_b32 s3, exec_lo, s3
-; GFX10-NEXT:    s_cbranch_execz .LBB29_10
-; GFX10-NEXT:  ; %bb.9:
-; GFX10-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-NEXT:    s_and_saveexec_b32 s2, vcc_lo
+; GFX10-NEXT:    s_xor_b32 s2, exec_lo, s2
+; GFX10-NEXT:    s_cbranch_execz .LBB29_8
+; GFX10-NEXT:  ; %bb.7:
+; GFX10-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX10-NEXT:    ds_add_rtn_f32 v2, v2, v1
-; GFX10-NEXT:  .LBB29_10:
+; GFX10-NEXT:  .LBB29_8:
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s3
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s2
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
@@ -8169,85 +8091,74 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
 ;
 ; GFX90A-LABEL: local_ds_fadd_one_as:
 ; GFX90A:       ; %bb.0:
+; GFX90A-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
 ; GFX90A-NEXT:    s_mov_b64 s[2:3], exec
-; GFX90A-NEXT:    v_bfrev_b32_e32 v1, 1
-; GFX90A-NEXT:    v_mov_b32_e32 v2, 0x42280000
-; GFX90A-NEXT:    ; implicit-def: $vgpr0
-; GFX90A-NEXT:  .LBB29_1: ; %ComputeLoop
-; GFX90A-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT:    s_ff1_i32_b64 s6, s[2:3]
-; GFX90A-NEXT:    s_lshl_b64 s[4:5], 1, s6
-; GFX90A-NEXT:    v_readfirstlane_b32 s7, v1
-; GFX90A-NEXT:    v_readlane_b32 s8, v2, s6
-; GFX90A-NEXT:    s_mov_b32 m0, s6
-; GFX90A-NEXT:    s_andn2_b64 s[2:3], s[2:3], s[4:5]
-; GFX90A-NEXT:    v_writelane_b32 v0, s7, m0
-; GFX90A-NEXT:    s_cmp_lg_u64 s[2:3], 0
-; GFX90A-NEXT:    v_add_f32_e32 v1, s8, v1
-; GFX90A-NEXT:    s_cbranch_scc1 .LBB29_1
-; GFX90A-NEXT:  ; %bb.2: ; %ComputeEnd
-; GFX90A-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
-; GFX90A-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
-; GFX90A-NEXT:    ; implicit-def: $vgpr2
+; GFX90A-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT:    s_add_i32 s3, s3, 4
-; GFX90A-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX90A-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX90A-NEXT:    s_add_i32 s5, s5, 4
+; GFX90A-NEXT:    ; implicit-def: $vgpr1
+; GFX90A-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; GFX90A-NEXT:    s_cbranch_execz .LBB29_2
+; GFX90A-NEXT:  ; %bb.1:
+; GFX90A-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
+; GFX90A-NEXT:    s_lshl_b32 s8, s5, 3
+; GFX90A-NEXT:    v_cvt_f32_ubyte0_e32 v1, s2
+; GFX90A-NEXT:    v_mul_f32_e32 v1, 0x42280000, v1
+; GFX90A-NEXT:    v_mov_b32_e32 v2, s8
+; GFX90A-NEXT:    ds_add_rtn_f32 v1, v2, v1
+; GFX90A-NEXT:  .LBB29_2:
+; GFX90A-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT:    s_mov_b64 s[8:9], exec
+; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX90A-NEXT:    v_readfirstlane_b32 s10, v1
+; GFX90A-NEXT:    v_mbcnt_lo_u32_b32 v1, s8, 0
+; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v1, s9, v1
+; GFX90A-NEXT:    v_cmp_eq_u32_e64 s[2:3], 0, v1
+; GFX90A-NEXT:    s_and_saveexec_b64 s[6:7], s[2:3]
 ; GFX90A-NEXT:    s_cbranch_execz .LBB29_4
 ; GFX90A-NEXT:  ; %bb.3:
-; GFX90A-NEXT:    s_lshl_b32 s6, s3, 3
-; GFX90A-NEXT:    v_mov_b32_e32 v2, s6
-; GFX90A-NEXT:    ds_add_rtn_f32 v2, v2, v1
-; GFX90A-NEXT:  .LBB29_4:
-; GFX90A-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT:    s_mov_b64 s[6:7], exec
-; GFX90A-NEXT:    v_mbcnt_lo_u32_b32 v1, s6, 0
-; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v1, s7, v1
-; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT:    v_readfirstlane_b32 s8, v2
-; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX90A-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX90A-NEXT:    s_cbranch_execz .LBB29_6
-; GFX90A-NEXT:  ; %bb.5:
-; GFX90A-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
-; GFX90A-NEXT:    v_cvt_f32_ubyte0_e32 v1, s6
-; GFX90A-NEXT:    s_lshl_b32 s3, s3, 4
+; GFX90A-NEXT:    s_bcnt1_i32_b64 s2, s[8:9]
+; GFX90A-NEXT:    v_cvt_f32_ubyte0_e32 v1, s2
+; GFX90A-NEXT:    s_lshl_b32 s2, s5, 4
 ; GFX90A-NEXT:    v_mul_f32_e32 v1, 0x42280000, v1
-; GFX90A-NEXT:    v_mov_b32_e32 v2, s3
+; GFX90A-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX90A-NEXT:    ds_add_f32 v2, v1
-; GFX90A-NEXT:  .LBB29_6:
-; GFX90A-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT:    s_mov_b64 s[4:5], exec
-; GFX90A-NEXT:    v_add_f32_e32 v2, s8, v0
+; GFX90A-NEXT:  .LBB29_4:
+; GFX90A-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
+; GFX90A-NEXT:    v_mul_f32_e32 v0, 0x42280000, v0
 ; GFX90A-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX90A-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX90A-NEXT:    s_mov_b64 s[2:3], exec
+; GFX90A-NEXT:    v_add_f32_e32 v2, s10, v0
 ; GFX90A-NEXT:    ; implicit-def: $vgpr0
-; GFX90A-NEXT:  .LBB29_7: ; %ComputeLoop1
+; GFX90A-NEXT:  .LBB29_5: ; %ComputeLoop
 ; GFX90A-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT:    s_ff1_i32_b64 s3, s[4:5]
-; GFX90A-NEXT:    s_lshl_b64 s[6:7], 1, s3
+; GFX90A-NEXT:    s_ff1_i32_b64 s5, s[2:3]
+; GFX90A-NEXT:    s_lshl_b64 s[6:7], 1, s5
 ; GFX90A-NEXT:    v_readfirstlane_b32 s8, v1
-; GFX90A-NEXT:    v_readlane_b32 s9, v2, s3
-; GFX90A-NEXT:    s_mov_b32 m0, s3
-; GFX90A-NEXT:    s_andn2_b64 s[4:5], s[4:5], s[6:7]
+; GFX90A-NEXT:    v_readlane_b32 s9, v2, s5
+; GFX90A-NEXT:    s_mov_b32 m0, s5
+; GFX90A-NEXT:    s_andn2_b64 s[2:3], s[2:3], s[6:7]
 ; GFX90A-NEXT:    v_writelane_b32 v0, s8, m0
-; GFX90A-NEXT:    s_cmp_lg_u64 s[4:5], 0
+; GFX90A-NEXT:    s_cmp_lg_u64 s[2:3], 0
 ; GFX90A-NEXT:    v_add_f32_e32 v1, s9, v1
-; GFX90A-NEXT:    s_cbranch_scc1 .LBB29_7
-; GFX90A-NEXT:  ; %bb.8: ; %ComputeEnd2
+; GFX90A-NEXT:    s_cbranch_scc1 .LBB29_5
+; GFX90A-NEXT:  ; %bb.6: ; %ComputeEnd
 ; GFX90A-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
 ; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v2, exec_hi, v2
 ; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
 ; GFX90A-NEXT:    ; implicit-def: $vgpr2
-; GFX90A-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX90A-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GFX90A-NEXT:    s_cbranch_execz .LBB29_10
-; GFX90A-NEXT:  ; %bb.9:
-; GFX90A-NEXT:    v_mov_b32_e32 v2, s2
+; GFX90A-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX90A-NEXT:    s_xor_b64 s[2:3], exec, s[2:3]
+; GFX90A-NEXT:    s_cbranch_execz .LBB29_8
+; GFX90A-NEXT:  ; %bb.7:
+; GFX90A-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX90A-NEXT:    ds_add_rtn_f32 v2, v2, v1
-; GFX90A-NEXT:  .LBB29_10:
-; GFX90A-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT:  .LBB29_8:
+; GFX90A-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
 ; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90A-NEXT:    v_readfirstlane_b32 s2, v2
@@ -8258,85 +8169,74 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
 ;
 ; GFX908-LABEL: local_ds_fadd_one_as:
 ; GFX908:       ; %bb.0:
+; GFX908-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
 ; GFX908-NEXT:    s_mov_b64 s[2:3], exec
-; GFX908-NEXT:    v_bfrev_b32_e32 v1, 1
-; GFX908-NEXT:    v_mov_b32_e32 v2, 0x42280000
-; GFX908-NEXT:    ; implicit-def: $vgpr0
-; GFX908-NEXT:  .LBB29_1: ; %ComputeLoop
-; GFX908-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT:    s_ff1_i32_b64 s6, s[2:3]
-; GFX908-NEXT:    s_lshl_b64 s[4:5], 1, s6
-; GFX908-NEXT:    v_readfirstlane_b32 s7, v1
-; GFX908-NEXT:    v_readlane_b32 s8, v2, s6
-; GFX908-NEXT:    s_mov_b32 m0, s6
-; GFX908-NEXT:    s_andn2_b64 s[2:3], s[2:3], s[4:5]
-; GFX908-NEXT:    v_writelane_b32 v0, s7, m0
-; GFX908-NEXT:    s_cmp_lg_u64 s[2:3], 0
-; GFX908-NEXT:    v_add_f32_e32 v1, s8, v1
-; GFX908-NEXT:    s_cbranch_scc1 .LBB29_1
-; GFX908-NEXT:  ; %bb.2: ; %ComputeEnd
-; GFX908-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
-; GFX908-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX908-NEXT:    v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX908-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
-; GFX908-NEXT:    ; implicit-def: $vgpr2
+; GFX908-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX908-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX908-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX908-NEXT:    s_add_i32 s3, s3, 4
-; GFX908-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX908-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX908-NEXT:    s_add_i32 s5, s5, 4
+; GFX908-NEXT:    ; implicit-def: $vgpr1
+; GFX908-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; GFX908-NEXT:    s_cbranch_execz .LBB29_2
+; GFX908-NEXT:  ; %bb.1:
+; GFX908-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
+; GFX908-NEXT:    s_lshl_b32 s8, s5, 3
+; GFX908-NEXT:    v_cvt_f32_ubyte0_e32 v1, s2
+; GFX908-NEXT:    v_mul_f32_e32 v1, 0x42280000, v1
+; GFX908-NEXT:    v_mov_b32_e32 v2, s8
+; GFX908-NEXT:    ds_add_rtn_f32 v1, v2, v1
+; GFX908-NEXT:  .LBB29_2:
+; GFX908-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX908-NEXT:    s_mov_b64 s[8:9], exec
+; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX908-NEXT:    v_readfirstlane_b32 s10, v1
+; GFX908-NEXT:    v_mbcnt_lo_u32_b32 v1, s8, 0
+; GFX908-NEXT:    v_mbcnt_hi_u32_b32 v1, s9, v1
+; GFX908-NEXT:    v_cmp_eq_u32_e64 s[2:3], 0, v1
+; GFX908-NEXT:    s_and_saveexec_b64 s[6:7], s[2:3]
 ; GFX908-NEXT:    s_cbranch_execz .LBB29_4
 ; GFX908-NEXT:  ; %bb.3:
-; GFX908-NEXT:    s_lshl_b32 s6, s3, 3
-; GFX908-NEXT:    v_mov_b32_e32 v2, s6
-; GFX908-NEXT:    ds_add_rtn_f32 v2, v2, v1
-; GFX908-NEXT:  .LBB29_4:
-; GFX908-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX908-NEXT:    s_mov_b64 s[6:7], exec
-; GFX908-NEXT:    v_mbcnt_lo_u32_b32 v1, s6, 0
-; GFX908-NEXT:    v_mbcnt_hi_u32_b32 v1, s7, v1
-; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX908-NEXT:    v_readfirstlane_b32 s8, v2
-; GFX908-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX908-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX908-NEXT:    s_cbranch_execz .LBB29_6
-; GFX908-NEXT:  ; %bb.5:
-; GFX908-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
-; GFX908-NEXT:    v_cvt_f32_ubyte0_e32 v1, s6
-; GFX908-NEXT:    s_lshl_b32 s3, s3, 4
+; GFX908-NEXT:    s_bcnt1_i32_b64 s2, s[8:9]
+; GFX908-NEXT:    v_cvt_f32_ubyte0_e32 v1, s2
+; GFX908-NEXT:    s_lshl_b32 s2, s5, 4
 ; GFX908-NEXT:    v_mul_f32_e32 v1, 0x42280000, v1
-; GFX908-NEXT:    v_mov_b32_e32 v2, s3
+; GFX908-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX908-NEXT:    ds_add_f32 v2, v1
-; GFX908-NEXT:  .LBB29_6:
-; GFX908-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX908-NEXT:    s_mov_b64 s[4:5], exec
-; GFX908-NEXT:    v_add_f32_e32 v2, s8, v0
+; GFX908-NEXT:  .LBB29_4:
+; GFX908-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX908-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
+; GFX908-NEXT:    v_mul_f32_e32 v0, 0x42280000, v0
 ; GFX908-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX908-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX908-NEXT:    s_mov_b64 s[2:3], exec
+; GFX908-NEXT:    v_add_f32_e32 v2, s10, v0
 ; GFX908-NEXT:    ; implicit-def: $vgpr0
-; GFX908-NEXT:  .LBB29_7: ; %ComputeLoop1
+; GFX908-NEXT:  .LBB29_5: ; %ComputeLoop
 ; GFX908-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT:    s_ff1_i32_b64 s3, s[4:5]
-; GFX908-NEXT:    s_lshl_b64 s[6:7], 1, s3
+; GFX908-NEXT:    s_ff1_i32_b64 s5, s[2:3]
+; GFX908-NEXT:    s_lshl_b64 s[6:7], 1, s5
 ; GFX908-NEXT:    v_readfirstlane_b32 s8, v1
-; GFX908-NEXT:    v_readlane_b32 s9, v2, s3
-; GFX908-NEXT:    s_mov_b32 m0, s3
-; GFX908-NEXT:    s_andn2_b64 s[4:5], s[4:5], s[6:7]
+; GFX908-NEXT:    v_readlane_b32 s9, v2, s5
+; GFX908-NEXT:    s_mov_b32 m0, s5
+; GFX908-NEXT:    s_andn2_b64 s[2:3], s[2:3], s[6:7]
 ; GFX908-NEXT:    v_writelane_b32 v0, s8, m0
-; GFX908-NEXT:    s_cmp_lg_u64 s[4:5], 0
+; GFX908-NEXT:    s_cmp_lg_u64 s[2:3], 0
 ; GFX908-NEXT:    v_add_f32_e32 v1, s9, v1
-; GFX908-NEXT:    s_cbranch_scc1 .LBB29_7
-; GFX908-NEXT:  ; %bb.8: ; %ComputeEnd2
+; GFX908-NEXT:    s_cbranch_scc1 .LBB29_5
+; GFX908-NEXT:  ; %bb.6: ; %ComputeEnd
 ; GFX908-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
 ; GFX908-NEXT:    v_mbcnt_hi_u32_b32 v2, exec_hi, v2
 ; GFX908-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
 ; GFX908-NEXT:    ; implicit-def: $vgpr2
-; GFX908-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX908-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GFX908-NEXT:    s_cbranch_execz .LBB29_10
-; GFX908-NEXT:  ; %bb.9:
-; GFX908-NEXT:    v_mov_b32_e32 v2, s2
+; GFX908-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX908-NEXT:    s_xor_b64 s[2:3], exec, s[2:3]
+; GFX908-NEXT:    s_cbranch_execz .LBB29_8
+; GFX908-NEXT:  ; %bb.7:
+; GFX908-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX908-NEXT:    ds_add_rtn_f32 v2, v2, v1
-; GFX908-NEXT:  .LBB29_10:
-; GFX908-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT:  .LBB29_8:
+; GFX908-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX908-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
 ; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX908-NEXT:    v_readfirstlane_b32 s2, v2
@@ -8347,87 +8247,76 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
 ;
 ; GFX8-LABEL: local_ds_fadd_one_as:
 ; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
 ; GFX8-NEXT:    s_mov_b64 s[2:3], exec
-; GFX8-NEXT:    v_bfrev_b32_e32 v1, 1
-; GFX8-NEXT:    v_mov_b32_e32 v2, 0x42280000
-; GFX8-NEXT:    ; implicit-def: $vgpr0
-; GFX8-NEXT:  .LBB29_1: ; %ComputeLoop
-; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT:    s_ff1_i32_b64 s6, s[2:3]
-; GFX8-NEXT:    s_lshl_b64 s[4:5], 1, s6
-; GFX8-NEXT:    v_readfirstlane_b32 s7, v1
-; GFX8-NEXT:    v_readlane_b32 s8, v2, s6
-; GFX8-NEXT:    s_mov_b32 m0, s6
-; GFX8-NEXT:    s_andn2_b64 s[2:3], s[2:3], s[4:5]
-; GFX8-NEXT:    v_writelane_b32 v0, s7, m0
-; GFX8-NEXT:    s_cmp_lg_u64 s[2:3], 0
-; GFX8-NEXT:    v_add_f32_e32 v1, s8, v1
-; GFX8-NEXT:    s_cbranch_scc1 .LBB29_1
-; GFX8-NEXT:  ; %bb.2: ; %ComputeEnd
-; GFX8-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
-; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX8-NEXT:    ; implicit-def: $vgpr2
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_add_i32 s5, s5, 4
+; GFX8-NEXT:    ; implicit-def: $vgpr1
 ; GFX8-NEXT:    s_mov_b32 m0, -1
+; GFX8-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; GFX8-NEXT:    s_cbranch_execz .LBB29_2
+; GFX8-NEXT:  ; %bb.1:
+; GFX8-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
+; GFX8-NEXT:    s_lshl_b32 s8, s5, 3
+; GFX8-NEXT:    v_cvt_f32_ubyte0_e32 v1, s2
+; GFX8-NEXT:    v_mul_f32_e32 v1, 0x42280000, v1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s8
+; GFX8-NEXT:    ds_add_rtn_f32 v1, v2, v1
+; GFX8-NEXT:  .LBB29_2:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX8-NEXT:    s_mov_b64 s[8:9], exec
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_add_i32 s3, s3, 4
-; GFX8-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX8-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GFX8-NEXT:    v_readfirstlane_b32 s10, v1
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v1, s8, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v1, s9, v1
+; GFX8-NEXT:    v_cmp_eq_u32_e64 s[2:3], 0, v1
+; GFX8-NEXT:    s_and_saveexec_b64 s[6:7], s[2:3]
 ; GFX8-NEXT:    s_cbranch_execz .LBB29_4
 ; GFX8-NEXT:  ; %bb.3:
-; GFX8-NEXT:    s_lshl_b32 s6, s3, 3
-; GFX8-NEXT:    v_mov_b32_e32 v2, s6
-; GFX8-NEXT:    ds_add_rtn_f32 v2, v2, v1
-; GFX8-NEXT:  .LBB29_4:
-; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX8-NEXT:    s_mov_b64 s[6:7], exec
-; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v1, s6, 0
-; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v1, s7, v1
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_readfirstlane_b32 s8, v2
-; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX8-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX8-NEXT:    s_cbranch_execz .LBB29_6
-; GFX8-NEXT:  ; %bb.5:
-; GFX8-NEXT:    s_bcnt1_i32_b64 s6, s[6:7]
-; GFX8-NEXT:    v_cvt_f32_ubyte0_e32 v1, s6
-; GFX8-NEXT:    s_lshl_b32 s3, s3, 4
+; GFX8-NEXT:    s_bcnt1_i32_b64 s2, s[8:9]
+; GFX8-NEXT:    v_cvt_f32_ubyte0_e32 v1, s2
+; GFX8-NEXT:    s_lshl_b32 s2, s5, 4
 ; GFX8-NEXT:    v_mul_f32_e32 v1, 0x42280000, v1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s3
+; GFX8-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX8-NEXT:    ds_add_f32 v2, v1
-; GFX8-NEXT:  .LBB29_6:
-; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX8-NEXT:    s_mov_b64 s[4:5], exec
-; GFX8-NEXT:    v_add_f32_e32 v2, s8, v0
+; GFX8-NEXT:  .LBB29_4:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX8-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
+; GFX8-NEXT:    v_mul_f32_e32 v0, 0x42280000, v0
 ; GFX8-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX8-NEXT:    s_mov_b64 s[2:3], exec
+; GFX8-NEXT:    v_add_f32_e32 v2, s10, v0
 ; GFX8-NEXT:    ; implicit-def: $vgpr0
-; GFX8-NEXT:  .LBB29_7: ; %ComputeLoop1
+; GFX8-NEXT:  .LBB29_5: ; %ComputeLoop
 ; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT:    s_ff1_i32_b64 s3, s[4:5]
-; GFX8-NEXT:    s_lshl_b64 s[6:7], 1, s3
+; GFX8-NEXT:    s_ff1_i32_b64 s5, s[2:3]
+; GFX8-NEXT:    s_lshl_b64 s[6:7], 1, s5
 ; GFX8-NEXT:    v_readfirstlane_b32 s8, v1
-; GFX8-NEXT:    v_readlane_b32 s9, v2, s3
-; GFX8-NEXT:    s_mov_b32 m0, s3
-; GFX8-NEXT:    s_andn2_b64 s[4:5], s[4:5], s[6:7]
+; GFX8-NEXT:    v_readlane_b32 s9, v2, s5
+; GFX8-NEXT:    s_mov_b32 m0, s5
+; GFX8-NEXT:    s_andn2_b64 s[2:3], s[2:3], s[6:7]
 ; GFX8-NEXT:    v_writelane_b32 v0, s8, m0
-; GFX8-NEXT:    s_cmp_lg_u64 s[4:5], 0
+; GFX8-NEXT:    s_cmp_lg_u64 s[2:3], 0
 ; GFX8-NEXT:    v_add_f32_e32 v1, s9, v1
-; GFX8-NEXT:    s_cbranch_scc1 .LBB29_7
-; GFX8-NEXT:  ; %bb.8: ; %ComputeEnd2
+; GFX8-NEXT:    s_cbranch_scc1 .LBB29_5
+; GFX8-NEXT:  ; %bb.6: ; %ComputeEnd
 ; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
 ; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v2, exec_hi, v2
 ; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
 ; GFX8-NEXT:    ; implicit-def: $vgpr2
-; GFX8-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX8-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GFX8-NEXT:    s_cbranch_execz .LBB29_10
-; GFX8-NEXT:  ; %bb.9:
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
+; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX8-NEXT:    s_xor_b64 s[2:3], exec, s[2:3]
+; GFX8-NEXT:    s_cbranch_execz .LBB29_8
+; GFX8-NEXT:  ; %bb.7:
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX8-NEXT:    s_mov_b32 m0, -1
 ; GFX8-NEXT:    ds_add_rtn_f32 v2, v2, v1
-; GFX8-NEXT:  .LBB29_10:
-; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT:  .LBB29_8:
+; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_readfirstlane_b32 s2, v2
@@ -8439,153 +8328,186 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
 ;
 ; GFX7-LABEL: local_ds_fadd_one_as:
 ; GFX7:       ; %bb.0:
-; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
-; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
+; GFX7-NEXT:    s_mov_b64 s[2:3], exec
+; GFX7-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX7-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_lshl_b32 s4, s3, 3
-; GFX7-NEXT:    v_mov_b32_e32 v0, s4
-; GFX7-NEXT:    ds_read_b32 v0, v0 offset:32
-; GFX7-NEXT:    s_add_i32 s3, s3, 4
-; GFX7-NEXT:    s_lshl_b32 s6, s3, 3
-; GFX7-NEXT:    s_mov_b64 s[4:5], 0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s6
-; GFX7-NEXT:  .LBB29_1: ; %atomicrmw.start
+; GFX7-NEXT:    s_add_i32 s5, s5, 4
+; GFX7-NEXT:    ; implicit-def: $vgpr1
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; GFX7-NEXT:    s_cbranch_execz .LBB29_4
+; GFX7-NEXT:  ; %bb.1:
+; GFX7-NEXT:    s_lshl_b32 s8, s5, 3
+; GFX7-NEXT:    v_mov_b32_e32 v2, s8
+; GFX7-NEXT:    ds_read_b32 v1, v2
+; GFX7-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
+; GFX7-NEXT:    v_cvt_f32_ubyte0_e32 v3, s2
+; GFX7-NEXT:    v_mul_f32_e32 v3, 0x42280000, v3
+; GFX7-NEXT:    s_mov_b64 s[8:9], 0
+; GFX7-NEXT:  .LBB29_2: ; %atomicrmw.start
 ; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v2, v0
-; GFX7-NEXT:    v_add_f32_e32 v0, 0x42280000, v2
-; GFX7-NEXT:    ds_cmpst_rtn_b32 v0, v1, v2, v0
+; GFX7-NEXT:    v_mov_b32_e32 v4, v1
+; GFX7-NEXT:    v_add_f32_e32 v1, v4, v3
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v2, v4, v1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
-; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT:    s_cbranch_execnz .LBB29_1
-; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT:    s_mov_b64 s[6:7], exec
-; GFX7-NEXT:    v_mbcnt_lo_u32_b32_e64 v1, s6, 0
-; GFX7-NEXT:    v_mbcnt_hi_u32_b32_e32 v1, s7, v1
-; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX7-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX7-NEXT:    s_cbranch_execz .LBB29_5
-; GFX7-NEXT:  ; %bb.3:
-; GFX7-NEXT:    s_lshl_b32 s3, s3, 4
-; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    v_cmp_eq_u32_e64 s[2:3], v1, v4
+; GFX7-NEXT:    s_or_b64 s[8:9], s[2:3], s[8:9]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; GFX7-NEXT:    s_cbranch_execnz .LBB29_2
+; GFX7-NEXT:  ; %bb.3: ; %Flow18
+; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
+; GFX7-NEXT:  .LBB29_4: ; %Flow19
+; GFX7-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX7-NEXT:    s_mov_b64 s[8:9], exec
+; GFX7-NEXT:    v_readfirstlane_b32 s10, v1
+; GFX7-NEXT:    v_mbcnt_lo_u32_b32_e64 v1, s8, 0
+; GFX7-NEXT:    v_mbcnt_hi_u32_b32_e32 v1, s9, v1
+; GFX7-NEXT:    v_cmp_eq_u32_e64 s[2:3], 0, v1
+; GFX7-NEXT:    s_and_saveexec_b64 s[6:7], s[2:3]
+; GFX7-NEXT:    s_cbranch_execz .LBB29_7
+; GFX7-NEXT:  ; %bb.5:
+; GFX7-NEXT:    s_lshl_b32 s2, s5, 4
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX7-NEXT:    ds_read_b32 v3, v1
-; GFX7-NEXT:    s_bcnt1_i32_b64 s3, s[6:7]
-; GFX7-NEXT:    v_cvt_f32_ubyte0_e32 v2, s3
+; GFX7-NEXT:    s_bcnt1_i32_b64 s2, s[8:9]
+; GFX7-NEXT:    v_cvt_f32_ubyte0_e32 v2, s2
 ; GFX7-NEXT:    v_mul_f32_e32 v2, 0x42280000, v2
-; GFX7-NEXT:    s_mov_b64 s[6:7], 0
-; GFX7-NEXT:  .LBB29_4: ; %atomicrmw.start2
+; GFX7-NEXT:    s_mov_b64 s[8:9], 0
+; GFX7-NEXT:  .LBB29_6: ; %atomicrmw.start2
 ; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    v_add_f32_e32 v4, v3, v2
 ; GFX7-NEXT:    ds_cmpst_rtn_b32 v4, v1, v3, v4
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX7-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7-NEXT:    v_cmp_eq_u32_e64 s[2:3], v4, v3
+; GFX7-NEXT:    s_or_b64 s[8:9], s[2:3], s[8:9]
 ; GFX7-NEXT:    v_mov_b32_e32 v3, v4
-; GFX7-NEXT:    s_andn2_b64 exec, exec, s[6:7]
-; GFX7-NEXT:    s_cbranch_execnz .LBB29_4
-; GFX7-NEXT:  .LBB29_5: ; %Flow17
-; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    ds_read_b32 v1, v2
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; GFX7-NEXT:    s_cbranch_execnz .LBB29_6
+; GFX7-NEXT:  .LBB29_7: ; %Flow17
+; GFX7-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX7-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s4
+; GFX7-NEXT:    v_mul_f32_e32 v2, 0x42280000, v0
+; GFX7-NEXT:    ds_read_b32 v0, v1
+; GFX7-NEXT:    v_bfrev_b32_e32 v3, 1
+; GFX7-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX7-NEXT:    v_add_f32_e32 v2, s10, v2
 ; GFX7-NEXT:    s_mov_b64 s[2:3], 0
-; GFX7-NEXT:  .LBB29_6: ; %atomicrmw.start8
+; GFX7-NEXT:  .LBB29_8: ; %atomicrmw.start8
 ; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v3, v1
-; GFX7-NEXT:    v_add_f32_e32 v1, v3, v0
-; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v2, v3, v1
+; GFX7-NEXT:    v_mov_b32_e32 v3, v0
+; GFX7-NEXT:    v_add_f32_e32 v0, v3, v2
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v0, v1, v3, v0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v3
 ; GFX7-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
 ; GFX7-NEXT:    s_andn2_b64 exec, exec, s[2:3]
-; GFX7-NEXT:    s_cbranch_execnz .LBB29_6
-; GFX7-NEXT:  ; %bb.7: ; %atomicrmw.end7
+; GFX7-NEXT:    s_cbranch_execnz .LBB29_8
+; GFX7-NEXT:  ; %bb.9: ; %atomicrmw.end7
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
 ; GFX7-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7-NEXT:    s_mov_b32 s2, -1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    buffer_store_dword v1, off, s[0:3], 0
+; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX6-LABEL: local_ds_fadd_one_as:
 ; GFX6:       ; %bb.0:
-; GFX6-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
-; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
+; GFX6-NEXT:    s_mov_b64 s[2:3], exec
+; GFX6-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX6-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    s_lshl_b32 s4, s3, 3
-; GFX6-NEXT:    s_add_i32 s4, s4, 32
-; GFX6-NEXT:    v_mov_b32_e32 v0, s4
-; GFX6-NEXT:    ds_read_b32 v0, v0
-; GFX6-NEXT:    s_add_i32 s3, s3, 4
-; GFX6-NEXT:    s_lshl_b32 s6, s3, 3
-; GFX6-NEXT:    s_mov_b64 s[4:5], 0
-; GFX6-NEXT:    v_mov_b32_e32 v1, s6
-; GFX6-NEXT:  .LBB29_1: ; %atomicrmw.start
+; GFX6-NEXT:    s_add_i32 s5, s5, 4
+; GFX6-NEXT:    ; implicit-def: $vgpr1
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; GFX6-NEXT:    s_cbranch_execz .LBB29_4
+; GFX6-NEXT:  ; %bb.1:
+; GFX6-NEXT:    s_lshl_b32 s8, s5, 3
+; GFX6-NEXT:    v_mov_b32_e32 v2, s8
+; GFX6-NEXT:    ds_read_b32 v1, v2
+; GFX6-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
+; GFX6-NEXT:    v_cvt_f32_ubyte0_e32 v3, s2
+; GFX6-NEXT:    v_mul_f32_e32 v3, 0x42280000, v3
+; GFX6-NEXT:    s_mov_b64 s[8:9], 0
+; GFX6-NEXT:  .LBB29_2: ; %atomicrmw.start
 ; GFX6-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    v_mov_b32_e32 v2, v0
-; GFX6-NEXT:    v_add_f32_e32 v0, 0x42280000, v2
-; GFX6-NEXT:    ds_cmpst_rtn_b32 v0, v1, v2, v0
+; GFX6-NEXT:    v_mov_b32_e32 v4, v1
+; GFX6-NEXT:    v_add_f32_e32 v1, v4, v3
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v2, v4, v1
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
-; GFX6-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX6-NEXT:    s_cbranch_execnz .LBB29_1
-; GFX6-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX6-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX6-NEXT:    s_mov_b64 s[6:7], exec
-; GFX6-NEXT:    v_mbcnt_lo_u32_b32_e64 v1, s6, 0
-; GFX6-NEXT:    v_mbcnt_hi_u32_b32_e32 v1, s7, v1
-; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX6-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX6-NEXT:    s_cbranch_execz .LBB29_5
-; GFX6-NEXT:  ; %bb.3:
-; GFX6-NEXT:    s_lshl_b32 s3, s3, 4
-; GFX6-NEXT:    v_mov_b32_e32 v1, s3
+; GFX6-NEXT:    v_cmp_eq_u32_e64 s[2:3], v1, v4
+; GFX6-NEXT:    s_or_b64 s[8:9], s[2:3], s[8:9]
+; GFX6-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; GFX6-NEXT:    s_cbranch_execnz .LBB29_2
+; GFX6-NEXT:  ; %bb.3: ; %Flow16
+; GFX6-NEXT:    s_or_b64 exec, exec, s[8:9]
+; GFX6-NEXT:  .LBB29_4: ; %Flow17
+; GFX6-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX6-NEXT:    s_mov_b64 s[8:9], exec
+; GFX6-NEXT:    v_readfirstlane_b32 s10, v1
+; GFX6-NEXT:    v_mbcnt_lo_u32_b32_e64 v1, s8, 0
+; GFX6-NEXT:    v_mbcnt_hi_u32_b32_e32 v1, s9, v1
+; GFX6-NEXT:    v_cmp_eq_u32_e64 s[2:3], 0, v1
+; GFX6-NEXT:    s_and_saveexec_b64 s[6:7], s[2:3]
+; GFX6-NEXT:    s_cbranch_execz .LBB29_7
+; GFX6-NEXT:  ; %bb.5:
+; GFX6-NEXT:    s_lshl_b32 s2, s5, 4
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX6-NEXT:    ds_read_b32 v3, v1
-; GFX6-NEXT:    s_bcnt1_i32_b64 s3, s[6:7]
-; GFX6-NEXT:    v_cvt_f32_ubyte0_e32 v2, s3
+; GFX6-NEXT:    s_bcnt1_i32_b64 s2, s[8:9]
+; GFX6-NEXT:    v_cvt_f32_ubyte0_e32 v2, s2
 ; GFX6-NEXT:    v_mul_f32_e32 v2, 0x42280000, v2
-; GFX6-NEXT:    s_mov_b64 s[6:7], 0
-; GFX6-NEXT:  .LBB29_4: ; %atomicrmw.start2
+; GFX6-NEXT:    s_mov_b64 s[8:9], 0
+; GFX6-NEXT:  .LBB29_6: ; %atomicrmw.start2
 ; GFX6-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX6-NEXT:    v_add_f32_e32 v4, v3, v2
 ; GFX6-NEXT:    ds_cmpst_rtn_b32 v4, v1, v3, v4
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX6-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GFX6-NEXT:    v_cmp_eq_u32_e64 s[2:3], v4, v3
+; GFX6-NEXT:    s_or_b64 s[8:9], s[2:3], s[8:9]
 ; GFX6-NEXT:    v_mov_b32_e32 v3, v4
-; GFX6-NEXT:    s_andn2_b64 exec, exec, s[6:7]
-; GFX6-NEXT:    s_cbranch_execnz .LBB29_4
-; GFX6-NEXT:  .LBB29_5: ; %Flow15
-; GFX6-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX6-NEXT:    v_mov_b32_e32 v2, s2
-; GFX6-NEXT:    ds_read_b32 v1, v2
+; GFX6-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; GFX6-NEXT:    s_cbranch_execnz .LBB29_6
+; GFX6-NEXT:  .LBB29_7: ; %Flow15
+; GFX6-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX6-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s4
+; GFX6-NEXT:    v_mul_f32_e32 v2, 0x42280000, v0
+; GFX6-NEXT:    ds_read_b32 v0, v1
+; GFX6-NEXT:    v_bfrev_b32_e32 v3, 1
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX6-NEXT:    v_add_f32_e32 v2, s10, v2
 ; GFX6-NEXT:    s_mov_b64 s[2:3], 0
-; GFX6-NEXT:  .LBB29_6: ; %atomicrmw.start8
+; GFX6-NEXT:  .LBB29_8: ; %atomicrmw.start8
 ; GFX6-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    v_mov_b32_e32 v3, v1
-; GFX6-NEXT:    v_add_f32_e32 v1, v3, v0
-; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v2, v3, v1
+; GFX6-NEXT:    v_mov_b32_e32 v3, v0
+; GFX6-NEXT:    v_add_f32_e32 v0, v3, v2
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v0, v1, v3, v0
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v3
 ; GFX6-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
 ; GFX6-NEXT:    s_andn2_b64 exec, exec, s[2:3]
-; GFX6-NEXT:    s_cbranch_execnz .LBB29_6
-; GFX6-NEXT:  ; %bb.7: ; %atomicrmw.end7
+; GFX6-NEXT:    s_cbranch_execnz .LBB29_8
+; GFX6-NEXT:  ; %bb.9: ; %atomicrmw.end7
 ; GFX6-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX6-NEXT:    s_mov_b32 s2, -1
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    buffer_store_dword v1, off, s[0:3], 0
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX6-NEXT:    s_endpgm
   %idx.add = add nuw i32 %idx, 4
   %shl0 = shl i32 %idx.add, 3



More information about the llvm-commits mailing list